Skip to content

Instantly share code, notes, and snippets.

@JasonYCHuang
Last active June 22, 2020 17:50
Show Gist options
  • Select an option

  • Save JasonYCHuang/3d85cf9b6f78ae36f4bcfa798ecdb188 to your computer and use it in GitHub Desktop.

Select an option

Save JasonYCHuang/3d85cf9b6f78ae36f4bcfa798ecdb188 to your computer and use it in GitHub Desktop.

Revisions

  1. JasonYCHuang revised this gist Jun 22, 2020. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions hadoop.md
    Original file line number Diff line number Diff line change
    @@ -17,8 +17,8 @@ alias hfcat="hf -cat" # Output a file to standard out
    alias hfchgrp="hf -chgrp" # Change group association of files
    alias hfchmod="hf -chmod" # Change permissions
    alias hfchown="hf -chown" # Change ownership
    alias hfcfl="hf -copyFromLocal" # Copy a local file reference to HDFS
    alias hfctl="hf -copyToLocal" # Copy a HDFS file reference to local
    alias hfcpfl="hf -copyFromLocal" # Copy a local file reference to HDFS
    alias hfcptl="hf -copyToLocal" # Copy a HDFS file reference to local
    alias hfcp="hf -cp" # Copy files from source to destination
    alias hfdu="hf -du" # Display aggregate length of files
    alias hfdus="hf -dus" # Display a summary of file lengths
  2. JasonYCHuang created this gist Nov 22, 2019.
    73 changes: 73 additions & 0 deletions hadoop.md
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,73 @@
    ```
    # Generic Aliases
    alias ll='ls -latr' # List all file in long list format by modification time
    alias ..='cd ..' # Go up one directory
    alias ...='cd ../..' # Go up two directories
    alias ....='cd ../../..' # Go up three directories
    alias -- -='cd -' # Go back
    alias c='clear' # Clear Screen
    alias k='clear' # Clear Screen
    alias cls='clear' # Clear Screen
    alias _="sudo" # Execute with sudo
    alias q='exit' # Logout
    # Common Hadoop File System Aliases
    alias hf="hadoop fs" # Base Hadoop fs command
    alias hfcat="hf -cat" # Output a file to standard out
    alias hfchgrp="hf -chgrp" # Change group association of files
    alias hfchmod="hf -chmod" # Change permissions
    alias hfchown="hf -chown" # Change ownership
    alias hfcfl="hf -copyFromLocal" # Copy a local file reference to HDFS
    alias hfctl="hf -copyToLocal" # Copy a HDFS file reference to local
    alias hfcp="hf -cp" # Copy files from source to destination
    alias hfdu="hf -du" # Display aggregate length of files
    alias hfdus="hf -dus" # Display a summary of file lengths
    alias hfget="hf -get" # Get a file from hadoop to local
    alias hfgetm="hf -getmerge" # Get files from hadoop to a local file
    alias hfls="hf -ls" # List files
    alias hfll="hf -lsr" # List files recursivly
    alias hfmkdir="hf -mkdir" # Make a directory
    alias hfmv="hf -mv" # Move a file
    alias hfput="hf -put" # Put a file from local to hadoop
    alias hfrm="hf -rm" # Remove a file
    alias hfrmr="hf -rmr" # Remove a file recursivly
    alias hfsr="hf -setrep" # Set the replication factor of a file
    alias hfstat="hf -stat" # Returns the stat information on the path
    alias hftail="hf -tail" # Tail a file
    alias hftest="hf -test" # Run a series of file tests. See options
    alias hftouch="hf -touchz" # Create a file of zero length
    alias hgrep="hdfs dfs -ls -R / | grep" # Search for a file in HDFS
    # Convenient Hadoop File System Aliases
    alias hfet="hf -rmr .Trash" # Remove/Empty the trash
    function hfdub() { # Display aggregate size of files descending
    hadoop fs -du "$@" | sort -k 1 -n -r
    }
    #Common Hadoop Job Commands
    alias hj="hadoop job" # Base Hadoop job command
    alias hjstat="hj -status" # Print completion percentage and all job counters
    alias hjkill="hj -kill" # Kills the job
    alias hjhist="hj -history" # Prints job details, failed and killed tip details
    alias hjlist="hj -list" # List jobs
    #Common Hadoop DFS Admin Commands
    alias habal="hadoop balancer" # Runs a cluster balancing utility
    alias harep="hadoop dfsadmin -report" # Print the hdfs admin report
    # Yarn Commands
    alias yl="yarn application -list" # List of Yarn Applications
    alias yk="yarn application -kill"
    #Common Oozie Aliases/Functions
    alias owv="oozie validate" # Validate a workflow xml
    alias ojrun="oozie job -run" # Run a job
    alias ojresume="oozie job -resume" # Resume a job
    alias ojrerun="oozie job -rerun" # Rerun a job
    alias ojsuspend="oozie job -suspend" # Suspend a job
    alias ojkill="oozie job -kill" # Kill a job
    alias ojinfo="oozie job -info" # Display current info on a job
    alias ojlist="oozie jobs -localtime" # Display a list of jobs
    alias ojlistr="oozie jobs -localtime -filter status=RUNNING" # Display a list of running jobs
    ```