ohbob · May 22, 2025 20:33
diff --git a/log2 b/log2
 #!/bin/bash

 # --- Defaults ---
 JOB_NAME=""
 NODE_ENV=""
 LOKI_URL="http://localhost:3100"
 LOG_LEVEL="debug"
 COMMAND_ARGS=()
 BUFFER_SIZE=100
 FLUSH_INTERVAL=5

 # --- Show help function ---
 show_help() {
  echo "log2 - Log forwarder to Loki with automatic log level detection"
  echo
  echo "USAGE:"
  echo "  log2 [OPTIONS] <command>"
  echo
  echo "OPTIONS:"
  echo "  --name NAME     Service name for log identification (default: directory name)"
  echo "  --env ENV       Environment name (default: dev)"
  echo "  --url URL       Loki server URL (default: http://localhost:3100)"
  echo "  --level LEVEL   Minimum log level to record (default: debug)"
  echo "                  Valid levels: debug, log, info, warn, error"
  echo "  --help          Show this help message"
  echo
  echo "EXAMPLES:"
  echo "  log2 python script.py                   # Run Python script with logs forwarded to Loki"
  echo "  log2 --name api node server.js          # Run Node.js with custom service name"
  echo "  log2 --env production ./myapp           # Run any command with environment specified"
  echo "  log2 --level warn --name db mysql       # Only capture warnings and errors"
  echo "  log2 --url http://loki.example.com:3100 --name frontend npm start"
  echo
  echo "NOTES:"
  echo "  - Python commands are automatically run in unbuffered mode"
  echo "  - Regular print/log statements appear as 'log' level (gray in Grafana)"
  echo "  - Info, warn, error, and debug levels are auto-detected from message content"
  echo "  - You can explicitly set log levels with prefixes: 'info: Your message'"
  echo
  exit 0
 }

 # --- Parse arguments ---
 while [[ $# -gt 0 ]]; do
  case $1 in
    --help)
      show_help
      ;;
    --name)
      JOB_NAME="$2"
      shift 2
      ;;
    --env)
      NODE_ENV="$2"
      shift 2
      ;;
    --url)
      LOKI_URL="$2"
      shift 2
      ;;
    --level)
      LOG_LEVEL="$2"
      shift 2
      ;;
    *)
      COMMAND_ARGS+=("$1")
      shift
      ;;
  esac
 done

 # --- Join command ---
 COMMAND="${COMMAND_ARGS[*]}"

 if [ -z "$COMMAND" ]; then
  show_help
 fi

 # --- Auto-detect name if missing ---
 if [ -z "$JOB_NAME" ]; then
  JOB_NAME=$(basename "$PWD")
 fi

 # --- Apply default values if missing ---
 NODE_ENV=${NODE_ENV:-"dev"}

 # --- Check if command is Python and handle buffering ---
 if [[ "$COMMAND" == python* || "$COMMAND" == *python3* ]]; then
  echo "[LOG2] Python detected - enabling unbuffered mode"
  
  # Set unbuffered mode for Python
  export PYTHONUNBUFFERED=1
  
  # If command starts with python/python3, add -u flag
  if [[ "$COMMAND" =~ ^python([0-9](\.[0-9]+)?)? ]]; then
    # Split command into an array
    IFS=' ' read -ra CMD_ARRAY <<< "$COMMAND"
    
    # Insert -u flag after python command
    UPDATED_CMD=("${CMD_ARRAY[0]}" "-u")
    if [ ${#CMD_ARRAY[@]} -gt 1 ]; then
      UPDATED_CMD+=("${CMD_ARRAY[@]:1}")
    fi
    
    # Join array back into command
    COMMAND=$(printf "%s " "${UPDATED_CMD[@]}")
    COMMAND=${COMMAND% }  # Remove trailing space
    echo "[LOG2] Command modified: $COMMAND"
  else
    echo "[LOG2] Python detected in command, using PYTHONUNBUFFERED=1"
  fi
 fi

 # --- Create temp files for log buffering ---
 BUFFER_FILE=$(mktemp)
 TIMESTAMP_FILE=$(mktemp)

 # Clean up temp files on exit
 trap 'rm -f $BUFFER_FILE $TIMESTAMP_FILE' EXIT

 # --- Color output functions ---
 color_log() { echo -e "\033[90m$1\033[0m"; }
 color_info() { echo -e "\033[90m$1\033[0m"; }
 color_warn() { echo -e "\033[33m$1\033[0m"; }
 color_error() { echo -e "\033[31m$1\033[0m"; }
 color_debug() { echo -e "\033[36m$1\033[0m"; }

 # --- Log level check ---
 should_log() {
  local level="$1"
  local levels=("debug" "log" "info" "warn" "error")
  local min_level_index=0
  
  # Find index of minimum log level
  for i in "${!levels[@]}"; do
    if [ "${levels[$i]}" = "$LOG_LEVEL" ]; then
      min_level_index=$i
      break
    fi
  done
  
  # Find index of current log level
  local current_level_index=0
  for i in "${!levels[@]}"; do
    if [ "${levels[$i]}" = "$level" ]; then
      current_level_index=$i
      break
    fi
  done
  
  # Return true if current level is >= minimum level
  [ $current_level_index -ge $min_level_index ]
  return $?
 }

 # --- Detect log level from message ---
 detect_log_level() {
  local msg="$1"
  local level="log"
  
  # Check for explicit prefixes
  if [[ "$msg" =~ ^[[:space:]]*(error|warn|info|debug|log):[[:space:]]+ ]]; then
    level="${BASH_REMATCH[1]}"
    return 0
  fi
  
  # Check for error patterns
  if [[ "$msg" =~ error|exception|fatal|critical|panic|crash|fail|abort|uncaught|unhandled|stack[[:space:]]*trace|segmentation[[:space:]]*fault|timeout|timed[[:space:]]*out ]]; then
    level="error"
  # Check for warning patterns
  elif [[ "$msg" =~ warn|warning|deprecated|obsolete|legacy|not[[:space:]]*found|missing|absent ]]; then
    level="warn"
  # Check for info patterns
  elif [[ "$msg" =~ start|started|starting|launch|launched|launching|boot|booting|booted|listen|server|connect|connected|load|loaded|loading|init|success|complete|completed|finish|finished ]]; then
    level="info"
  fi
  
  echo "$level"
 }

 # --- Add log to buffer ---
 add_log() {
  local level="$1"
  local msg="$2"
  
  # Skip if below minimum log level
  should_log "$level" || return 0
  
  # Format output with color
  case "$level" in
    log) color_log "$level: $msg" ;;
    info) color_info "$level: $msg" ;;
    warn) color_warn "$level: $msg" ;;
    error) color_error "$level: $msg" ;;
    debug) color_debug "$level: $msg" ;;
    *) echo "$level: $msg" ;;
  esac
  
  # Add to buffer
  local timestamp=$(date +%s%N)
  echo "$timestamp $level $msg" >> "$BUFFER_FILE"
  
  # Count lines in buffer
  local buffer_lines=$(wc -l < "$BUFFER_FILE")
  
  # Flush if buffer is full
  if [ "$buffer_lines" -ge "$BUFFER_SIZE" ]; then
    flush_logs
  fi
 }

 # --- Flush logs to Loki ---
 flush_logs() {
  # Skip if buffer is empty
  [ -s "$BUFFER_FILE" ] || return 0
  
  #echo "[LOG2] Flushing logs to $LOKI_URL..."
  
  # Create timestamp for batch
  local now=$(date +%s)
  
  # Prepare payload
  echo "{\"streams\": [{\"stream\": {\"job\": \"$JOB_NAME\", \"env\": \"$NODE_ENV\"}, \"values\": [" > "$TIMESTAMP_FILE"
  
  # Add log entries
  local first=true
  while IFS= read -r line; do
    local ts=$(echo "$line" | cut -d' ' -f1)
    local level=$(echo "$line" | cut -d' ' -f2)
    local content=$(echo "$line" | cut -d' ' -f3-)
    
    if [ "$first" = true ]; then
      first=false
    else
      echo "," >> "$TIMESTAMP_FILE"
    fi
    
    echo "[\"$ts\", \"$level: $content\"]" >> "$TIMESTAMP_FILE"
  done < "$BUFFER_FILE"
  
  echo "]}]}" >> "$TIMESTAMP_FILE"
  
  # Send to Loki using curl
  if command -v curl &> /dev/null; then
    curl -s -X POST -H "Content-Type: application/json" \
         --data-binary @"$TIMESTAMP_FILE" \
         "$LOKI_URL/loki/api/v1/push" > /dev/null || \
    echo "[LOG2] Failed to send logs to Loki"
  else
    echo "[LOG2] Warning: curl not found, logs not sent to Loki"
  fi
  
  # Clear buffer
  > "$BUFFER_FILE"
 }

 # --- Start background flush timer ---
 (
  while true; do
    sleep $FLUSH_INTERVAL
    flush_logs
  done
 ) &
 TIMER_PID=$!

 # Ensure timer is killed on exit
 trap 'kill $TIMER_PID 2>/dev/null; rm -f $BUFFER_FILE $TIMESTAMP_FILE' EXIT

 echo "[LOG2] Running: $COMMAND"
 echo "[LOG2] Logging $JOB_NAME ($NODE_ENV) to $LOKI_URL with level $LOG_LEVEL"

 # --- Run command and process output ---
 $COMMAND 2>&1 | while IFS= read -r line; do
  # Skip empty lines
  [ -z "$line" ] && continue
  
  # Extract level and message
  if [[ "$line" =~ ^[[:space:]]*(error|warn|info|debug|log):[[:space:]]+(.*)$ ]]; then
    level="${BASH_REMATCH[1]}"
    msg="${BASH_REMATCH[2]}"
  else
    level=$(detect_log_level "$line")
    msg="$line"
  fi
  
  add_log "$level" "$msg"
 done

 # Flush remaining logs
 flush_logs

 exit 0
	#!/bin/bash

	# --- Defaults ---
	JOB_NAME=""
	NODE_ENV=""
	LOKI_URL="http://localhost:3100"
	LOG_LEVEL="debug"
	COMMAND_ARGS=()
	BUFFER_SIZE=100
	FLUSH_INTERVAL=5

	# --- Show help function ---
	show_help() {
	echo "log2 - Log forwarder to Loki with automatic log level detection"
	echo
	echo "USAGE:"
	echo " log2 [OPTIONS] <command>"
	echo
	echo "OPTIONS:"
	echo " --name NAME Service name for log identification (default: directory name)"
	echo " --env ENV Environment name (default: dev)"
	echo " --url URL Loki server URL (default: http://localhost:3100)"
	echo " --level LEVEL Minimum log level to record (default: debug)"
	echo " Valid levels: debug, log, info, warn, error"
	echo " --help Show this help message"
	echo
	echo "EXAMPLES:"
	echo " log2 python script.py # Run Python script with logs forwarded to Loki"
	echo " log2 --name api node server.js # Run Node.js with custom service name"
	echo " log2 --env production ./myapp # Run any command with environment specified"
	echo " log2 --level warn --name db mysql # Only capture warnings and errors"
	echo " log2 --url http://loki.example.com:3100 --name frontend npm start"
	echo
	echo "NOTES:"
	echo " - Python commands are automatically run in unbuffered mode"
	echo " - Regular print/log statements appear as 'log' level (gray in Grafana)"
	echo " - Info, warn, error, and debug levels are auto-detected from message content"
	echo " - You can explicitly set log levels with prefixes: 'info: Your message'"
	echo
	exit 0
	}

	# --- Parse arguments ---
	while [[ $# -gt 0 ]]; do
	case $1 in
	--help)
	show_help
	;;
	--name)
	JOB_NAME="$2"
	shift 2
	;;
	--env)
	NODE_ENV="$2"
	shift 2
	;;
	--url)
	LOKI_URL="$2"
	shift 2
	;;
	--level)
	LOG_LEVEL="$2"
	shift 2
	;;
	*)
	COMMAND_ARGS+=("$1")
	shift
	;;
	esac
	done

	# --- Join command ---
	COMMAND="${COMMAND_ARGS[*]}"

	if [ -z "$COMMAND" ]; then
	show_help
	fi

	# --- Auto-detect name if missing ---
	if [ -z "$JOB_NAME" ]; then
	JOB_NAME=$(basename "$PWD")
	fi

	# --- Apply default values if missing ---
	NODE_ENV=${NODE_ENV:-"dev"}

	# --- Check if command is Python and handle buffering ---
	if [[ "$COMMAND" == python* \|\| "$COMMAND" == python3 ]]; then
	echo "[LOG2] Python detected - enabling unbuffered mode"

	# Set unbuffered mode for Python
	export PYTHONUNBUFFERED=1

	# If command starts with python/python3, add -u flag
	if [[ "$COMMAND" =~ ^python([0-9](\.[0-9]+)?)? ]]; then
	# Split command into an array
	IFS=' ' read -ra CMD_ARRAY <<< "$COMMAND"

	# Insert -u flag after python command
	UPDATED_CMD=("${CMD_ARRAY[0]}" "-u")
	if [ ${#CMD_ARRAY[@]} -gt 1 ]; then
	UPDATED_CMD+=("${CMD_ARRAY[@]:1}")
	fi

	# Join array back into command
	COMMAND=$(printf "%s " "${UPDATED_CMD[@]}")
	COMMAND=${COMMAND% } # Remove trailing space
	echo "[LOG2] Command modified: $COMMAND"
	else
	echo "[LOG2] Python detected in command, using PYTHONUNBUFFERED=1"
	fi
	fi

	# --- Create temp files for log buffering ---
	BUFFER_FILE=$(mktemp)
	TIMESTAMP_FILE=$(mktemp)

	# Clean up temp files on exit
	trap 'rm -f $BUFFER_FILE $TIMESTAMP_FILE' EXIT

	# --- Color output functions ---
	color_log() { echo -e "\033[90m$1\033[0m"; }
	color_info() { echo -e "\033[90m$1\033[0m"; }
	color_warn() { echo -e "\033[33m$1\033[0m"; }
	color_error() { echo -e "\033[31m$1\033[0m"; }
	color_debug() { echo -e "\033[36m$1\033[0m"; }

	# --- Log level check ---
	should_log() {
	local level="$1"
	local levels=("debug" "log" "info" "warn" "error")
	local min_level_index=0

	# Find index of minimum log level
	for i in "${!levels[@]}"; do
	if [ "${levels[$i]}" = "$LOG_LEVEL" ]; then
	min_level_index=$i
	break
	fi
	done

	# Find index of current log level
	local current_level_index=0
	for i in "${!levels[@]}"; do
	if [ "${levels[$i]}" = "$level" ]; then
	current_level_index=$i
	break
	fi
	done

	# Return true if current level is >= minimum level
	[ $current_level_index -ge $min_level_index ]
	return $?
	}

	# --- Detect log level from message ---
	detect_log_level() {
	local msg="$1"
	local level="log"

	# Check for explicit prefixes
	if [[ "$msg" =~ ^[[:space:]]*(error\|warn\|info\|debug\|log):[[:space:]]+ ]]; then
	level="${BASH_REMATCH[1]}"
	return 0
	fi

	# Check for error patterns
	if [[ "$msg" =~ error\|exception\|fatal\|critical\|panic\|crash\|fail\|abort\|uncaught\|unhandled\|stack[[:space:]]trace\|segmentation[[:space:]]fault\|timeout\|timed[[:space:]]*out ]]; then
	level="error"
	# Check for warning patterns
	elif [[ "$msg" =~ warn\|warning\|deprecated\|obsolete\|legacy\|not[[:space:]]*found\|missing\|absent ]]; then
	level="warn"
	# Check for info patterns
	elif [[ "$msg" =~ start\|started\|starting\|launch\|launched\|launching\|boot\|booting\|booted\|listen\|server\|connect\|connected\|load\|loaded\|loading\|init\|success\|complete\|completed\|finish\|finished ]]; then
	level="info"
	fi

	echo "$level"
	}

	# --- Add log to buffer ---
	add_log() {
	local level="$1"
	local msg="$2"

	# Skip if below minimum log level
	should_log "$level" \|\| return 0

	# Format output with color
	case "$level" in
	log) color_log "$level: $msg" ;;
	info) color_info "$level: $msg" ;;
	warn) color_warn "$level: $msg" ;;
	error) color_error "$level: $msg" ;;
	debug) color_debug "$level: $msg" ;;
	*) echo "$level: $msg" ;;
	esac

	# Add to buffer
	local timestamp=$(date +%s%N)
	echo "$timestamp $level $msg" >> "$BUFFER_FILE"

	# Count lines in buffer
	local buffer_lines=$(wc -l < "$BUFFER_FILE")

	# Flush if buffer is full
	if [ "$buffer_lines" -ge "$BUFFER_SIZE" ]; then
	flush_logs
	fi
	}

	# --- Flush logs to Loki ---
	flush_logs() {
	# Skip if buffer is empty
	[ -s "$BUFFER_FILE" ] \|\| return 0

	#echo "[LOG2] Flushing logs to $LOKI_URL..."

	# Create timestamp for batch
	local now=$(date +%s)

	# Prepare payload
	echo "{\"streams\": [{\"stream\": {\"job\": \"$JOB_NAME\", \"env\": \"$NODE_ENV\"}, \"values\": [" > "$TIMESTAMP_FILE"

	# Add log entries
	local first=true
	while IFS= read -r line; do
	local ts=$(echo "$line" \| cut -d' ' -f1)
	local level=$(echo "$line" \| cut -d' ' -f2)
	local content=$(echo "$line" \| cut -d' ' -f3-)

	if [ "$first" = true ]; then
	first=false
	else
	echo "," >> "$TIMESTAMP_FILE"
	fi

	echo "[\"$ts\", \"$level: $content\"]" >> "$TIMESTAMP_FILE"
	done < "$BUFFER_FILE"

	echo "]}]}" >> "$TIMESTAMP_FILE"

	# Send to Loki using curl
	if command -v curl &> /dev/null; then
	curl -s -X POST -H "Content-Type: application/json" \
	--data-binary @"$TIMESTAMP_FILE" \
	"$LOKI_URL/loki/api/v1/push" > /dev/null \|\| \
	echo "[LOG2] Failed to send logs to Loki"
	else
	echo "[LOG2] Warning: curl not found, logs not sent to Loki"
	fi

	# Clear buffer
	> "$BUFFER_FILE"
	}

	# --- Start background flush timer ---
	(
	while true; do
	sleep $FLUSH_INTERVAL
	flush_logs
	done
	) &
	TIMER_PID=$!

	# Ensure timer is killed on exit
	trap 'kill $TIMER_PID 2>/dev/null; rm -f $BUFFER_FILE $TIMESTAMP_FILE' EXIT

	echo "[LOG2] Running: $COMMAND"
	echo "[LOG2] Logging $JOB_NAME ($NODE_ENV) to $LOKI_URL with level $LOG_LEVEL"

	# --- Run command and process output ---
	$COMMAND 2>&1 \| while IFS= read -r line; do
	# Skip empty lines
	[ -z "$line" ] && continue

	# Extract level and message
	if [[ "$line" =~ ^[[:space:]](error\|warn\|info\|debug\|log):[[:space:]]+(.)$ ]]; then
	level="${BASH_REMATCH[1]}"
	msg="${BASH_REMATCH[2]}"
	else
	level=$(detect_log_level "$line")
	msg="$line"
	fi

	add_log "$level" "$msg"
	done

	# Flush remaining logs
	flush_logs

	exit 0
No results found