Skip to content

Instantly share code, notes, and snippets.

@rdkls
rdkls / gist:b3dcbd5c725b50275f8df73a73f29baa
Created December 18, 2024 04:03
tgw flow logs athena table create, parquet format, hive partitions
-- src https://docs.aws.amazon.com/athena/latest/ug/vpc-flow-logs-parquet.html
CREATE EXTERNAL TABLE IF NOT EXISTS tgw_flow_logs (
version int,
account_id string,
interface_id string,
srcaddr string,
dstaddr string,
srcport int,
dstport int,
protocol bigint,
@rdkls
rdkls / aws-sso-profiles-gen.sh
Created November 3, 2023 03:27
from aws sso pages, generate profile entries for ~/.aws/config
#!/bin/bash
PORTALBASE=https://portal.sso.ap-southeast-2.amazonaws.com
REGION=ap-southeast-2
headers="-H 'X-Amz-Sso-Bearer-Token: $TOKEN' -H 'x-amz-sso_bearer_token: $TOKEN'"
if [ -z "$TOKEN" ]; then
echo "Must set $TOKEN before running. Get it from browser login to sso portal, and check header for x-amz-sso_bearer_token"
return 1
fi
# Define the provider and region
provider "aws" {
region = "ap-southeast-2"
}
variable "username" {
}
variable "password" {}
# Get the default VPC
data "aws_vpc" "default" {
@rdkls
rdkls / tfsec-json-to-gitlab-sast-report.py
Last active September 6, 2023 18:52
python script to convert tfsec json output into gitlab sast report, will get parsed by gitlab and result in vulnerabilities being visible/manageable in gitlab vuln management interface
#!/usr/bin/env python3
import sys
import datetime
import json
import uuid
'''
# Description
Convert tfsec json output to gitlab sast json format
TFSec https://github.com/aquasecurity/tfsec
@rdkls
rdkls / tls-ciphers.sh
Created July 12, 2023 06:22
list tls ciphers supported by a host
#!/bin/bash
# Check if the host argument is provided
if [ -z "$1" ]; then
echo "Usage: $0 host"
exit 1
fi
# Get the host name
host=$1
CREATE EXTERNAL TABLE IF NOT EXISTS vpc_flow_logs (
version int,
account string,
interfaceid string,
sourceaddress string,
destinationaddress string,
sourceport int,
destinationport int,
protocol int,
numpackets int,
@rdkls
rdkls / athena-nfw-to-neo4j.py
Created July 11, 2023 12:18
query athena nfw logs and insert into neo4j
#!/usr/bin/env python3
# run neo4j locally with e.g. docker run --rm -ti -p 7474:7474 -p7687:7687 --env NEO4J_AUTH=none neo4j
# athena table should be created per https://gist.github.com/rdkls/4d2228795b3a64d9a728f94e1441222f
from pyathena import connect
from neo4j import GraphDatabase
import threading
from halo import Halo # Import halo lib
# Define connection parameters
@rdkls
rdkls / nfw logs athena table create
Last active July 13, 2023 00:28
aws nfw firewall logs athena setup
https://docs.aws.amazon.com/network-firewall/latest/developerguide/firewall-logging.html
BUT add "alert" data which is super important when trying to troubleshoot which rule is blocking
CREATE EXTERNAL TABLE `nfw_logs`(
`firewall_name` string COMMENT 'from deserializer',
`availability_zone` string COMMENT 'from deserializer',
`event_timestamp` bigint COMMENT 'from deserializer',
`event` struct<
timestamp:string,
flow_id:bigint,
# brew install session-manager-plugin
# Bootstrap your ssh key to ec2 instance, then start ssh session over ssm to it
# Since it's plain ssh can then e.g. port forward and all the funky stuff
Host i-* mi-* ssm-jumphost
User ec2-user
IdentityFile ~/.ssh/nick-doyle-slalom-lab-ed25519
ProxyCommand sh -c "aws ec2-instance-connect send-ssh-public-key --instance-id %h --ssh-public-key '$(cat ~/.ssh/nick-doyle-slalom-lab-ed25519.pub)' --instance-os-user ec2-user; aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters 'portNumber=%p'"
# Fix Github Co-pilot self-signed cert problem
# See: https://github.com/orgs/community/discussions/8866#discussioncomment-3517831
# The script is modified to handle .vscode-server too (in WSL2)
fix_github_copilot() {
patch_ext() {
_VSCODEDIR=$1
_EXTENSIONSPATH="$HOME/${_VSCODEDIR}/extensions"
_RE=$2