Day 2 — Linux & Shell Essentials: The Survival Toolkit#

pwd
# /home/alice/projects/tds

ls                   # basic list
ls -l                # long format (permissions, size, date)
ls -a                # show hidden files (starting with .)
ls -la               # combine both — most useful
ls -lh               # human-readable file sizes (KB, MB, GB)
ls -lt               # sort by modification time, newest first
ls -lS               # sort by file size, largest first
ls -R                # recursive — list all subdirectories too
ls *.py              # list only .py files (glob pattern)
ls -la ~/projects/   # list a specific directory without cd-ing there

ls -la
# total 48
# drwxr-xr-x  4 alice alice 4096 Jan 15 10:23 .
# drwxr-xr-x 12 alice alice 4096 Jan 15 09:00 ..
# -rw-r--r--  1 alice alice  234 Jan 15 10:23 README.md
# -rwxr-xr-x  1 alice alice  891 Jan 14 18:45 run.sh
# drwxr-xr-x  2 alice alice 4096 Jan 14 15:30 data
# lrwxrwxrwx  1 alice alice   12 Jan 13 11:00 latest -> data/v2/
#  ^            ^^^^^  ^^^^^  ^^^  ^^^^^^^^^^^^  ^^^^^^^^^^^
#  |            owner  group  size  date          name
#  type+permissions
#
# First character:
# d = directory
# - = regular file
# l = symlink (→ shows where it points)

cd /home/alice/projects   # absolute path
cd projects               # relative — works if projects/ is in current dir
cd ..                     # parent directory
cd ../..                  # two levels up
cd ~                      # home directory
cd -                      # back to previous directory (like browser Back)
cd                        # no argument = same as cd ~

cd ~/projects/tds/day-2/data/
# do some work...
cd /tmp
# now go back:
cd -
# /home/alice/projects/tds/day-2/data/  ← returned

sudo apt install tree -y

# Basic usage:
tree
# .
# ├── data
# │   ├── raw.csv
# │   └── processed.csv
# ├── notebooks
# │   └── analysis.ipynb
# └── README.md

# Limit depth (very useful for large projects):
tree -L 2

# Show hidden files:
tree -a

# Show only directories:
tree -d

# Show file sizes:
tree -sh

# Output to file (useful for documentation):
tree -L 2 > structure.txt

# Create your full workspace structure in one command:
mkdir -p ~/tds-bootcamp/{day-1,day-2,day-3,day-4,day-5}
# The -p flag: create parent dirs as needed, no error if exists
# The {d1,d2} syntax: brace expansion — creates multiple at once

# Verify:
tree ~/tds-bootcamp/
# ~/tds-bootcamp/
# ├── day-1
# ├── day-2
# ├── day-3
# ├── day-4
# └── day-5

# Create nested structure:
mkdir -p ~/tds-bootcamp/day-2/{data,outputs,scripts}
tree ~/tds-bootcamp/day-2/

touch notes.txt              # create empty file
touch file1.txt file2.txt    # create multiple files
touch -t 202401011200 old.txt  # create with specific timestamp

# Also used to update the modification time of an existing file:
touch existing.txt    # updates timestamp without changing content

cp source.txt destination.txt          # copy file
cp source.txt /home/alice/backup/      # copy to directory
cp -r sourcedir/ destdir/              # copy directory recursively (-r is required)
cp -i source.txt dest.txt              # interactive — asks before overwriting
cp -v source.txt dest.txt              # verbose — shows what it copied
cp *.csv /backup/csvfiles/             # copy all .csv files

# Preserve permissions and timestamps:
cp -p source.txt dest.txt
cp -rp sourcedir/ destdir/             # recursive + preserve

# Copying a directory WITHOUT -r fails:
cp myproject/ backup/
# cp: -r not specified; omitting directory 'myproject/'

# Fix:
cp -r myproject/ backup/

mv oldname.txt newname.txt             # rename file
mv file.txt ~/projects/                # move to directory
mv *.log /var/archive/logs/            # move all .log files
mv -i source.txt dest.txt             # ask before overwrite
mv -v file.txt ~/backup/               # verbose

# Rename a directory:
mv old-project-name/ new-project-name/

# Move and rename simultaneously:
mv ~/Downloads/report-v1.pdf ~/projects/tds/report-final.pdf

rm file.txt                      # delete a file
rm file1.txt file2.txt           # delete multiple files
rm *.tmp                         # delete all .tmp files
rm -r directory/                 # delete directory and all contents
rm -i file.txt                   # interactive — asks for each file
rm -v file.txt                   # verbose — shows what's deleted

# DANGER ZONE:
rm -rf /path/to/directory/       # force recursive delete, no prompts
# -f = force (no error if file doesn't exist, no prompts)
# NEVER run: rm -rf /    or    rm -rf ~    or    rm -rf ./

mkdir newdir                     # create directory
mkdir -p a/b/c/d                 # create full path
mkdir -p project/{src,tests,docs,data}   # brace expansion

rmdir emptydir/                  # remove empty directory
rm -r nonemptydir/               # remove directory with contents

# * matches anything (zero or more characters)
ls *.txt            # all .txt files
rm temp_*.csv       # all files starting with temp_ ending in .csv

# ? matches exactly one character
ls file?.txt        # matches file1.txt, fileA.txt, NOT file10.txt

# [abc] matches one character from the set
ls file[123].txt    # matches file1.txt, file2.txt, file3.txt
ls [A-Z]*.py        # Python files starting with uppercase

# ** recursive glob (in some shells/tools)
ls **/*.py          # all .py files in any subdirectory

cat file.txt                     # print whole file
cat -n file.txt                  # print with line numbers
cat file1.txt file2.txt          # concatenate and print both
cat file1.txt file2.txt > combined.txt  # merge two files

less bigfile.txt
# Once inside less:
# Space / f      → next page
# b              → previous page
# /searchterm    → search forward (press n for next match)
# ?searchterm    → search backward
# G              → jump to end
# g              → jump to beginning
# q              → quit
# :n             → open next file (if multiple given)

less +F logfile.log    # follow mode — like tail -f, appends as file grows
                       # Ctrl+C to stop following, q to quit

head file.txt            # first 10 lines (default)
head -5 file.txt         # first 5 lines
head -c 100 file.txt     # first 100 bytes

tail file.txt            # last 10 lines
tail -20 file.txt        # last 20 lines
tail -c 500 file.txt     # last 500 bytes

# MOST IMPORTANT tail flag:
tail -f server.log       # follow — prints new lines as they appear
                         # Use this to watch a log file live
                         # Ctrl+C to stop

wc file.txt              # lines  words  chars  filename
wc -l file.txt           # line count only
wc -w file.txt           # word count only
wc -c file.txt           # byte count
wc -m file.txt           # character count (handles unicode correctly)

# Count files in a directory:
ls /usr/bin/ | wc -l

# Redirect stdout to a file (overwrites):
ls > filelist.txt

# Redirect stdout to a file (appends):
echo "new line" >> notes.txt

# Redirect stderr to a file:
python3 buggy.py 2> errors.txt

# Redirect both stdout and stderr to same file:
python3 script.py > output.txt 2>&1
# The 2>&1 means: "redirect stderr (2) to wherever stdout (1) is going"

# Separate stdout and stderr:
python3 script.py > output.txt 2> errors.txt

# Discard stderr (silence it):
python3 noisy.py 2>/dev/null

# Discard ALL output:
python3 script.py > /dev/null 2>&1

# Redirect stdin — feed a file as input:
python3 processor.py < input.txt

# Here-string — feed a string as stdin:
python3 -c "import sys; print(sys.stdin.read())" <<< "hello world"

# Basic pipe
ls -la | wc -l          # count total items in directory

# Count Python files in a project:
find . -name "*.py" | wc -l

# Without xargs — this is WRONG for many programs:
find . -name "*.tmp" | rm       # ERROR: rm doesn't read from stdin

# With xargs — CORRECT:
find . -name "*.tmp" | xargs rm

# -I{} lets you place the argument anywhere:
find . -name "*.txt" | xargs -I{} cp {} /backup/

# Process one at a time with -n 1:
echo "a b c" | xargs -n 1 echo

# Show what would be done (dry run):
find . -name "*.tmp" | xargs echo

# Define a variable (no spaces around =):
NAME="alice"
echo $NAME         # alice
echo ${NAME}       # alice (curly braces = safer, especially in strings)
echo "${NAME}_log" # alice_log  ← without braces: $NAME_log looks for $NAME_log var

# Variable types (bash is loosely typed):
COUNT=5
SUM=$((COUNT + 3))    # arithmetic: use $(( ))
echo $SUM             # 8

# Command output into a variable:
TODAY=$(date +%Y-%m-%d)
echo $TODAY            # 2024-01-15

PYTHON_VER=$(python3 --version)
echo $PYTHON_VER       # Python 3.11.2

# Read-only variable:
readonly PI=3.14159
PI=3                   # Error: PI: readonly variable

# if / else:
if [ condition ]; then
    commands
elif [ other_condition ]; then
    other_commands
else
    fallback_commands
fi

# Common conditions:
[ -f file.txt ]       # true if file exists and is a regular file
[ -d mydir ]          # true if directory exists
[ -z "$VAR" ]         # true if variable is empty
[ -n "$VAR" ]         # true if variable is non-empty
[ "$A" = "$B" ]       # string equality
[ "$A" != "$B" ]      # string inequality
[ $NUM -gt 10 ]       # numeric: greater than (-lt, -ge, -le, -eq, -ne)

# Example:
if [ -f "config.json" ]; then
    echo "Config found"
else
    echo "Config missing — creating default"
    cp config.default.json config.json
fi

# For loop over list:
for fruit in apple banana cherry; do
    echo "I like $fruit"
done

# For loop over files:
for file in *.csv; do
    echo "Processing $file..."
    wc -l "$file"
done

# For loop with range:
for i in {1..5}; do
    echo "Step $i"
done

# While loop:
count=0
while [ $count -lt 5 ]; do
    echo "count is $count"
    count=$((count + 1))
done

# While with input:
while read -r line; do
    echo "Got: $line"
done < input.txt

#!/bin/bash
# Day 2 workspace setup script
# Usage: ./setup-workspace.sh <project-name>

PROJECT_NAME="${1:?Usage: $0 <project-name>}"
BASE_DIR="$HOME/tds-bootcamp"
PROJECT_DIR="$BASE_DIR/$PROJECT_NAME"

echo "Creating project: $PROJECT_NAME"

# Create directory structure
mkdir -p "$PROJECT_DIR"/{data,outputs,scripts,docs}

# Create README
cat > "$PROJECT_DIR/README.md" << EOF
# $PROJECT_NAME
Created: $(date +%Y-%m-%d)

## Structure
- data/     : input data files
- outputs/  : generated files
- scripts/  : shell scripts
- docs/     : documentation
EOF

echo "Done! Project created at: $PROJECT_DIR"
echo ""
tree "$PROJECT_DIR"

chmod +x setup-workspace.sh
./setup-workspace.sh my-analysis

if [ -f "config.json" ]; then
    echo "Found"
else
    echo "Missing"
fi

# Almost every command supports --help:
ls --help
grep --help | head -30      # pipe to head if it's too long
find --help 2>&1 | less     # pipe to less for paging

man ls              # open manual for ls
man grep
man bash            # full bash reference (very long)

# Inside man:
# Space / f   → next page
# b           → back a page
# /term       → search
# q           → quit

# Search across all man pages:
man -k "copy file"     # find man pages related to copying files
apropos copy           # same as man -k

# Install:
sudo apt install tldr -y
tldr --update    # download the database

# Use:
tldr ls          # practical examples for ls
tldr grep        # common grep patterns
tldr tar         # tar is notoriously hard to remember
# tar: xkcd.com/1168

type ls            # ls is aliased to 'ls --color=auto'
type cd            # cd is a shell builtin
type python3       # python3 is /usr/bin/python3

which python3      # /usr/bin/python3
which -a python3   # show ALL matches in PATH, not just the first

info coreutils     # more detailed documentation (GNU tools)

mkdir -p /tmp/lab-workspace/{data,scripts,backups}

echo "id,name,role" > /tmp/lab-workspace/data/raw.csv
echo "1,alice,engineer" >> /tmp/lab-workspace/data/raw.csv
echo "2,bob,designer" >> /tmp/lab-workspace/data/raw.csv

3,carol,manager

#!/bin/bash
SOURCE_DIR="/tmp/lab-workspace/data"
DEST_DIR="/tmp/lab-workspace/backups"

if [ -d "$DEST_DIR" ]; then
    echo "Backup directory exists."
else
    echo "Backup directory missing. Creating..."
    mkdir -p "$DEST_DIR"
fi

echo "Starting backup..."
for file in $SOURCE_DIR/*.csv; do
    cp "$file" "$DEST_DIR/"
    echo "Copied $file to $DEST_DIR/"
done
echo "Backup complete!"

chmod +x /tmp/lab-workspace/scripts/backup.sh
/tmp/lab-workspace/scripts/backup.sh

wc -l /tmp/lab-workspace/backups/raw.csv

# Navigation
pwd / ls -la / cd / cd - / tree -L 2

# File operations
touch / cp -r / mv / rm -r / mkdir -p

# Reading files
cat -n / less / head -5 / tail -f / wc -l

# Pipelines and Redirection
|  >  >>  2>  2>&1  2>/dev/null  xargs

# Text Editors
nano / vim

# Shell Variables and Scripts
NAME="value" / $((expression)) / $(command) / [ -d dir ] / [ -f file ] / for x in list; do ... done

# Help
command --help / man command / tldr command / which command