%%bash
echo "TODAY'S DATE:"
date
echo "------------"
echo ""
#Display operating system info
lsb_release -a
echo ""
echo "------------"
echo "HOSTNAME: "; hostname
echo ""
echo "------------"
echo "Computer Specs:"
echo ""
lscpu
echo ""
echo "------------"
echo ""
echo "Memory Specs"
echo ""
free -mh
TODAY'S DATE: Wed Dec 12 13:26:25 PST 2018 ------------ Distributor ID: Ubuntu Description: Ubuntu 16.04.5 LTS Release: 16.04 Codename: xenial ------------ HOSTNAME: emu ------------ Computer Specs: Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Byte Order: Little Endian CPU(s): 16 On-line CPU(s) list: 0-15 Thread(s) per core: 2 Core(s) per socket: 4 Socket(s): 2 NUMA node(s): 1 Vendor ID: GenuineIntel CPU family: 6 Model: 26 Model name: Intel(R) Xeon(R) CPU E5520 @ 2.27GHz Stepping: 5 CPU MHz: 1596.000 CPU max MHz: 2394.0000 CPU min MHz: 1596.0000 BogoMIPS: 4521.80 Virtualization: VT-x L1d cache: 32K L1i cache: 32K L2 cache: 256K L3 cache: 8192K NUMA node0 CPU(s): 0-15 Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm dca sse4_1 sse4_2 popcnt lahf_lm tpr_shadow vnmi flexpriority ept vpid dtherm ida ------------ Memory Specs total used free shared buff/cache available Mem: 47G 1.5G 408M 125M 45G 44G Swap: 11G 22M 11G
No LSB modules are available.
%%bash
mkdir /home/sam/data/metagenomics
mkdir /home/sam/data/metagenomics/trimmed
mkdir /home/sam/analyses/20181211_metagenomics_fastqc_trimgalore
mkdir /home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_fastqc
mkdir /home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_01
mkdir /home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_01/20181211_metagenomics_trimmed_fastqc
mkdir /home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_02
mkdir /home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_02/20181211_metagenomics_trimmed_fastqc
mkdir: cannot create directory ‘/home/sam/data/metagenomics’: File exists mkdir: cannot create directory ‘/home/sam/data/metagenomics/trimmed’: File exists
%%bash
cd /home/sam/data/metagenomics
# Uncomment following lines to download files from server
#time \
#wget \
#--quiet \
#--no-directories \
#--recursive \
#--accept gz \
#--accept-regex "Library_Geoduck_MG_[0-9]" \
#http://owl.fish.washington.edu/nightingales/P_generosa/
time \
rsync --archive \
owl:/volume1/web/nightingales/P_generosa/Library_Geoduck*gz .
sed '/^Subject:/ s/ / metagenomics download JOB COMPLETE/' ~/.default-subject.mail | msmtp "$EMAIL"
ls -ltrh
total 86G -rwxr-xr-x 1 sam sam 7.2G Aug 8 13:10 Library_Geoduck_MG_3_S1_L002_R1_001.fastq.gz -rw-r--r-- 1 sam sam 7.1G Aug 8 13:10 Library_Geoduck_MG_1_S3_L002_R1_001.fastq.gz -rwxr-xr-x 1 sam sam 8.2G Aug 8 13:10 Library_Geoduck_MG_7_S2_L002_R2_001.fastq.gz -rwxr-xr-x 1 sam sam 7.3G Aug 8 13:10 Library_Geoduck_MG_7_S2_L002_R1_001.fastq.gz -rwxr-xr-x 1 sam sam 7.8G Aug 8 13:10 Library_Geoduck_MG_3_S1_L002_R2_001.fastq.gz -rwxr-xr-x 1 sam sam 6.8G Aug 8 13:10 Library_Geoduck_MG_2_S4_L002_R1_001.fastq.gz -rwxr-xr-x 1 sam sam 6.1G Aug 8 13:10 Library_Geoduck_MG_6_S5_L002_R1_001.fastq.gz -rwxr-xr-x 1 sam sam 7.9G Aug 8 13:10 Library_Geoduck_MG_1_S3_L002_R2_001.fastq.gz -rwxr-xr-x 1 sam sam 6.7G Aug 8 13:10 Library_Geoduck_MG_5_S6_L002_R2_001.fastq.gz -rwxr-xr-x 1 sam sam 7.7G Aug 8 13:10 Library_Geoduck_MG_2_S4_L002_R2_001.fastq.gz -rwxr-xr-x 1 sam sam 6.9G Aug 8 13:10 Library_Geoduck_MG_6_S5_L002_R2_001.fastq.gz -rwxr-xr-x 1 sam sam 6.0G Aug 8 13:10 Library_Geoduck_MG_5_S6_L002_R1_001.fastq.gz drwxrwxr-x 2 sam sam 4.0K Dec 11 13:27 trimmed
real 60m16.704s user 20m34.552s sys 7m59.060s
The code belows creates a space-delimited list of the FastQ files (FASTQ_LIST).
This is then passed to FastQC.
%%bash
fastq_dir="/home/sam/data/metagenomics/"
out_dir="/home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_fastqc"
cd ${fastq_dir}
# List all gzip files and then replace newline with space
FASTQ_LIST="$(ls -1 *.gz| tr '\n' ' ')"
# Run FastQC
time \
/home/shared/fastqc_0.11.7/fastqc \
--extract \
--threads 16 \
--quiet \
--outdir ${out_dir} \
$FASTQ_LIST
sed '/^Subject:/ s/ / fastqc JOB COMPLETE/' ~/.default-subject.mail | msmtp "$EMAIL"
ls ${out_dir}
Library_Geoduck_MG_1_S3_L002_R1_001_fastqc Library_Geoduck_MG_1_S3_L002_R1_001_fastqc.html Library_Geoduck_MG_1_S3_L002_R1_001_fastqc.zip Library_Geoduck_MG_1_S3_L002_R2_001_fastqc Library_Geoduck_MG_1_S3_L002_R2_001_fastqc.html Library_Geoduck_MG_1_S3_L002_R2_001_fastqc.zip Library_Geoduck_MG_2_S4_L002_R1_001_fastqc Library_Geoduck_MG_2_S4_L002_R1_001_fastqc.html Library_Geoduck_MG_2_S4_L002_R1_001_fastqc.zip Library_Geoduck_MG_2_S4_L002_R2_001_fastqc Library_Geoduck_MG_2_S4_L002_R2_001_fastqc.html Library_Geoduck_MG_2_S4_L002_R2_001_fastqc.zip Library_Geoduck_MG_3_S1_L002_R1_001_fastqc Library_Geoduck_MG_3_S1_L002_R1_001_fastqc.html Library_Geoduck_MG_3_S1_L002_R1_001_fastqc.zip Library_Geoduck_MG_3_S1_L002_R2_001_fastqc Library_Geoduck_MG_3_S1_L002_R2_001_fastqc.html Library_Geoduck_MG_3_S1_L002_R2_001_fastqc.zip Library_Geoduck_MG_5_S6_L002_R1_001_fastqc Library_Geoduck_MG_5_S6_L002_R1_001_fastqc.html Library_Geoduck_MG_5_S6_L002_R1_001_fastqc.zip Library_Geoduck_MG_5_S6_L002_R2_001_fastqc Library_Geoduck_MG_5_S6_L002_R2_001_fastqc.html Library_Geoduck_MG_5_S6_L002_R2_001_fastqc.zip Library_Geoduck_MG_6_S5_L002_R1_001_fastqc Library_Geoduck_MG_6_S5_L002_R1_001_fastqc.html Library_Geoduck_MG_6_S5_L002_R1_001_fastqc.zip Library_Geoduck_MG_6_S5_L002_R2_001_fastqc Library_Geoduck_MG_6_S5_L002_R2_001_fastqc.html Library_Geoduck_MG_6_S5_L002_R2_001_fastqc.zip Library_Geoduck_MG_7_S2_L002_R1_001_fastqc Library_Geoduck_MG_7_S2_L002_R1_001_fastqc.html Library_Geoduck_MG_7_S2_L002_R1_001_fastqc.zip Library_Geoduck_MG_7_S2_L002_R2_001_fastqc Library_Geoduck_MG_7_S2_L002_R2_001_fastqc.html Library_Geoduck_MG_7_S2_L002_R2_001_fastqc.zip
real 41m55.107s user 337m9.560s sys 5m10.576s
%%bash
cd /home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_fastqc
time \
multiqc .
ls
Searching 228 files.. Library_Geoduck_MG_1_S3_L002_R1_001_fastqc Library_Geoduck_MG_1_S3_L002_R1_001_fastqc.html Library_Geoduck_MG_1_S3_L002_R1_001_fastqc.zip Library_Geoduck_MG_1_S3_L002_R2_001_fastqc Library_Geoduck_MG_1_S3_L002_R2_001_fastqc.html Library_Geoduck_MG_1_S3_L002_R2_001_fastqc.zip Library_Geoduck_MG_2_S4_L002_R1_001_fastqc Library_Geoduck_MG_2_S4_L002_R1_001_fastqc.html Library_Geoduck_MG_2_S4_L002_R1_001_fastqc.zip Library_Geoduck_MG_2_S4_L002_R2_001_fastqc Library_Geoduck_MG_2_S4_L002_R2_001_fastqc.html Library_Geoduck_MG_2_S4_L002_R2_001_fastqc.zip Library_Geoduck_MG_3_S1_L002_R1_001_fastqc Library_Geoduck_MG_3_S1_L002_R1_001_fastqc.html Library_Geoduck_MG_3_S1_L002_R1_001_fastqc.zip Library_Geoduck_MG_3_S1_L002_R2_001_fastqc Library_Geoduck_MG_3_S1_L002_R2_001_fastqc.html Library_Geoduck_MG_3_S1_L002_R2_001_fastqc.zip Library_Geoduck_MG_5_S6_L002_R1_001_fastqc Library_Geoduck_MG_5_S6_L002_R1_001_fastqc.html Library_Geoduck_MG_5_S6_L002_R1_001_fastqc.zip Library_Geoduck_MG_5_S6_L002_R2_001_fastqc Library_Geoduck_MG_5_S6_L002_R2_001_fastqc.html Library_Geoduck_MG_5_S6_L002_R2_001_fastqc.zip Library_Geoduck_MG_6_S5_L002_R1_001_fastqc Library_Geoduck_MG_6_S5_L002_R1_001_fastqc.html Library_Geoduck_MG_6_S5_L002_R1_001_fastqc.zip Library_Geoduck_MG_6_S5_L002_R2_001_fastqc Library_Geoduck_MG_6_S5_L002_R2_001_fastqc.html Library_Geoduck_MG_6_S5_L002_R2_001_fastqc.zip Library_Geoduck_MG_7_S2_L002_R1_001_fastqc Library_Geoduck_MG_7_S2_L002_R1_001_fastqc.html Library_Geoduck_MG_7_S2_L002_R1_001_fastqc.zip Library_Geoduck_MG_7_S2_L002_R2_001_fastqc Library_Geoduck_MG_7_S2_L002_R2_001_fastqc.html Library_Geoduck_MG_7_S2_L002_R2_001_fastqc.zip multiqc_data multiqc_report.html
[INFO ] multiqc : This is MultiQC v1.6.dev0 [INFO ] multiqc : Template : default [INFO ] multiqc : Searching '.' [INFO ] fastqc : Found 12 reports [INFO ] multiqc : Compressing plot data [INFO ] multiqc : Report : multiqc_report.html [INFO ] multiqc : Data : multiqc_data [INFO ] multiqc : MultiQC complete real 0m8.683s user 0m4.724s sys 0m1.568s
%%bash
fastq_dir="/home/sam/data/metagenomics/"
trim_out_dir="/home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_01"
fastqc_out_dir="/home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_01/20181211_metagenomics_trimmed_fastqc"
cd ${fastq_dir}
# List all gzip files and then replace newline with space
FASTQ_LIST="$(ls -1 *.gz| tr '\n' ' ')"
# Run TrimGalore! with FastQC
time \
/home/shared/TrimGalore-0.4.5/trim_galore \
--paired \
--output_dir ${trim_out_dir} \
--fastqc_args \
"--outdir ${fastqc_out_dir} \
--threads 16" \
${FASTQ_LIST} \
> ${trim_out_dir}/20181211_trimgalore.stdout.log \
2> ${trim_out_dir}/20181211_trimgalore.stderr.log
# Run MultiQC
cd ${fastqc_out_dir}
multiqc .
sed '/^Subject:/ s/ / trimgalore JOB COMPLETE/' ~/.default-subject.mail | msmtp "$EMAIL"
echo ""
echo "--------------------"
echo "TAIL STDOUT"
echo "--------------------"
tail ${trim_out_dir}/20181211_trimgalore.stdout.log
echo ""
echo "--------------------"
echo "TAIL STDERR"
echo "--------------------"
tail ${trim_out_dir}/20181211_trimgalore.stderr.log
echo "--------------------"
echo ""
echo "--------------------"
echo ""
echo "TRIMGALORE LISTING"
cd ${trim_out_dir} && ls -lhtr ${trim_out_dir}
echo ""
echo "--------------------"
echo ""
echo "--------------------"
echo ""
echo "TRIMMED FASTQC LISTING"
cd ${fastqc_out_dir} && ls -lhtr ${fastqc_out_dir}
echo ""
echo "--------------------"
Searching 24 files.. -------------------- TAIL STDOUT -------------------- Analysis complete for Library_Geoduck_MG_2_S4_L002_R1_001_val_1.fq.gz Analysis complete for Library_Geoduck_MG_2_S4_L002_R2_001_val_2.fq.gz Analysis complete for Library_Geoduck_MG_3_S1_L002_R1_001_val_1.fq.gz Analysis complete for Library_Geoduck_MG_3_S1_L002_R2_001_val_2.fq.gz Analysis complete for Library_Geoduck_MG_5_S6_L002_R1_001_val_1.fq.gz Analysis complete for Library_Geoduck_MG_5_S6_L002_R2_001_val_2.fq.gz Analysis complete for Library_Geoduck_MG_6_S5_L002_R1_001_val_1.fq.gz Analysis complete for Library_Geoduck_MG_6_S5_L002_R2_001_val_2.fq.gz Analysis complete for Library_Geoduck_MG_7_S2_L002_R1_001_val_1.fq.gz Analysis complete for Library_Geoduck_MG_7_S2_L002_R2_001_val_2.fq.gz -------------------- TAIL STDERR -------------------- Approx 80% complete for Library_Geoduck_MG_7_S2_L002_R2_001_val_2.fq.gz Approx 85% complete for Library_Geoduck_MG_7_S2_L002_R2_001_val_2.fq.gz Approx 90% complete for Library_Geoduck_MG_7_S2_L002_R2_001_val_2.fq.gz Approx 95% complete for Library_Geoduck_MG_7_S2_L002_R2_001_val_2.fq.gz Deleting both intermediate output files Library_Geoduck_MG_7_S2_L002_R1_001_trimmed.fq.gz and Library_Geoduck_MG_7_S2_L002_R2_001_trimmed.fq.gz ==================================================================================================== gzip: stdout: Broken pipe -------------------- -------------------- TRIMGALORE LISTING total 71G -rw-rw-r-- 1 sam sam 5.7K Dec 12 15:04 Library_Geoduck_MG_1_S3_L002_R1_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 6.0G Dec 12 16:43 Library_Geoduck_MG_1_S3_L002_R1_001_val_1.fq.gz -rw-rw-r-- 1 sam sam 6.6G Dec 12 16:43 Library_Geoduck_MG_1_S3_L002_R2_001_val_2.fq.gz -rw-rw-r-- 1 sam sam 6.0K Dec 12 16:43 Library_Geoduck_MG_1_S3_L002_R2_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.8K Dec 12 18:11 Library_Geoduck_MG_2_S4_L002_R1_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.8G Dec 12 19:48 Library_Geoduck_MG_2_S4_L002_R1_001_val_1.fq.gz -rw-rw-r-- 1 sam sam 6.4G Dec 12 19:48 Library_Geoduck_MG_2_S4_L002_R2_001_val_2.fq.gz -rw-rw-r-- 1 sam sam 6.0K Dec 12 19:48 Library_Geoduck_MG_2_S4_L002_R2_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.8K Dec 12 21:13 Library_Geoduck_MG_3_S1_L002_R1_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.7G Dec 12 22:46 Library_Geoduck_MG_3_S1_L002_R1_001_val_1.fq.gz -rw-rw-r-- 1 sam sam 6.1G Dec 12 22:46 Library_Geoduck_MG_3_S1_L002_R2_001_val_2.fq.gz -rw-rw-r-- 1 sam sam 6.0K Dec 12 22:46 Library_Geoduck_MG_3_S1_L002_R2_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.7K Dec 13 00:07 Library_Geoduck_MG_5_S6_L002_R1_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.1G Dec 13 01:32 Library_Geoduck_MG_5_S6_L002_R1_001_val_1.fq.gz -rw-rw-r-- 1 sam sam 5.6G Dec 13 01:32 Library_Geoduck_MG_5_S6_L002_R2_001_val_2.fq.gz -rw-rw-r-- 1 sam sam 5.9K Dec 13 01:32 Library_Geoduck_MG_5_S6_L002_R2_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.7K Dec 13 02:49 Library_Geoduck_MG_6_S5_L002_R1_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.2G Dec 13 04:16 Library_Geoduck_MG_6_S5_L002_R1_001_val_1.fq.gz -rw-rw-r-- 1 sam sam 5.7G Dec 13 04:16 Library_Geoduck_MG_6_S5_L002_R2_001_val_2.fq.gz -rw-rw-r-- 1 sam sam 5.9K Dec 13 04:16 Library_Geoduck_MG_6_S5_L002_R2_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.8K Dec 13 05:42 Library_Geoduck_MG_7_S2_L002_R1_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 6.2G Dec 13 07:25 Library_Geoduck_MG_7_S2_L002_R1_001_val_1.fq.gz -rw-rw-r-- 1 sam sam 6.8G Dec 13 07:25 Library_Geoduck_MG_7_S2_L002_R2_001_val_2.fq.gz -rw-rw-r-- 1 sam sam 6.0K Dec 13 07:25 Library_Geoduck_MG_7_S2_L002_R2_001.fastq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 845 Dec 13 08:08 20181211_trimgalore.stdout.log -rw-rw-r-- 1 sam sam 103K Dec 13 08:08 20181211_trimgalore.stderr.log drwxrwxr-x 3 sam sam 4.0K Dec 13 08:08 20181211_metagenomics_trimmed_fastqc -------------------- -------------------- TRIMMED FASTQC LISTING total 7.7M -rw-rw-r-- 1 sam sam 300K Dec 12 17:04 Library_Geoduck_MG_1_S3_L002_R1_001_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 242K Dec 12 17:04 Library_Geoduck_MG_1_S3_L002_R1_001_val_1_fastqc.html -rw-rw-r-- 1 sam sam 310K Dec 12 17:25 Library_Geoduck_MG_1_S3_L002_R2_001_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 247K Dec 12 17:25 Library_Geoduck_MG_1_S3_L002_R2_001_val_2_fastqc.html -rw-rw-r-- 1 sam sam 305K Dec 12 20:08 Library_Geoduck_MG_2_S4_L002_R1_001_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 245K Dec 12 20:08 Library_Geoduck_MG_2_S4_L002_R1_001_val_1_fastqc.html -rw-rw-r-- 1 sam sam 307K Dec 12 20:28 Library_Geoduck_MG_2_S4_L002_R2_001_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 245K Dec 12 20:28 Library_Geoduck_MG_2_S4_L002_R2_001_val_2_fastqc.html -rw-rw-r-- 1 sam sam 305K Dec 12 23:06 Library_Geoduck_MG_3_S1_L002_R1_001_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 244K Dec 12 23:06 Library_Geoduck_MG_3_S1_L002_R1_001_val_1_fastqc.html -rw-rw-r-- 1 sam sam 307K Dec 12 23:26 Library_Geoduck_MG_3_S1_L002_R2_001_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 246K Dec 12 23:26 Library_Geoduck_MG_3_S1_L002_R2_001_val_2_fastqc.html -rw-rw-r-- 1 sam sam 303K Dec 13 01:50 Library_Geoduck_MG_5_S6_L002_R1_001_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 244K Dec 13 01:50 Library_Geoduck_MG_5_S6_L002_R1_001_val_1_fastqc.html -rw-rw-r-- 1 sam sam 307K Dec 13 02:07 Library_Geoduck_MG_5_S6_L002_R2_001_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 246K Dec 13 02:07 Library_Geoduck_MG_5_S6_L002_R2_001_val_2_fastqc.html -rw-rw-r-- 1 sam sam 303K Dec 13 04:34 Library_Geoduck_MG_6_S5_L002_R1_001_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 244K Dec 13 04:34 Library_Geoduck_MG_6_S5_L002_R1_001_val_1_fastqc.html -rw-rw-r-- 1 sam sam 308K Dec 13 04:52 Library_Geoduck_MG_6_S5_L002_R2_001_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 246K Dec 13 04:52 Library_Geoduck_MG_6_S5_L002_R2_001_val_2_fastqc.html -rw-rw-r-- 1 sam sam 301K Dec 13 07:46 Library_Geoduck_MG_7_S2_L002_R1_001_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 242K Dec 13 07:46 Library_Geoduck_MG_7_S2_L002_R1_001_val_1_fastqc.html -rw-rw-r-- 1 sam sam 309K Dec 13 08:08 Library_Geoduck_MG_7_S2_L002_R2_001_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 246K Dec 13 08:08 Library_Geoduck_MG_7_S2_L002_R2_001_val_2_fastqc.html -rw-rw-r-- 1 sam sam 1.2M Dec 13 08:08 multiqc_report.html drwxrwxr-x 2 sam sam 4.0K Dec 13 08:08 multiqc_data --------------------
real 1072m12.860s user 1953m48.580s sys 36m26.676s [INFO ] multiqc : This is MultiQC v1.6.dev0 [INFO ] multiqc : Template : default [INFO ] multiqc : Searching '.' [INFO ] fastqc : Found 12 reports [INFO ] multiqc : Compressing plot data [INFO ] multiqc : Report : multiqc_report.html [INFO ] multiqc : Data : multiqc_data [INFO ] multiqc : MultiQC complete
%%bash
trimmed_fastq_dir="/home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_01"
trim_out_dir="/home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_02"
fastqc_out_dir="/home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_02/20181211_metagenomics_trimmed_fastqc"
cd ${trimmed_fastq_dir}
# List all gzip files and then replace newline with space
FASTQ_LIST="$(ls -1 *.gz| tr '\n' ' ')"
# Run TrimGalore! with FastQC
# Trim first 10bp from each read.
time \
/home/shared/TrimGalore-0.4.5/trim_galore \
--paired \
--output_dir ${trim_out_dir} \
--clip_R1 10 \
--clip_R2 10 \
--fastqc_args \
"--outdir ${fastqc_out_dir} \
--threads 16" \
${FASTQ_LIST} \
> ${trim_out_dir}/20181211_trimgalore.stdout.log \
2> ${trim_out_dir}/20181211_trimgalore.stderr.log
# Run MultiQC
cd ${fastqc_out_dir}
multiqc .
sed '/^Subject:/ s/ / trimgalore JOB COMPLETE/' ~/.default-subject.mail | msmtp "$EMAIL"
echo ""
echo "--------------------"
echo "TAIL STDOUT"
echo "--------------------"
tail ${trim_out_dir}/20181211_trimgalore.stdout.log
echo ""
echo "--------------------"
echo "TAIL STDERR"
echo "--------------------"
tail ${trim_out_dir}/20181211_trimgalore.stderr.log
echo "--------------------"
echo ""
echo "--------------------"
echo ""
echo "TRIMGALORE LISTING"
cd ${trim_out_dir} && ls -lhtr ${trim_out_dir}
echo ""
echo "--------------------"
echo ""
echo "--------------------"
echo ""
echo "TRIMMED FASTQC LISTING"
cd ${fastqc_out_dir} && ls -lhtr ${fastqc_out_dir}
echo ""
echo "--------------------"
Searching 24 files.. -------------------- TAIL STDOUT -------------------- Analysis complete for Library_Geoduck_MG_2_S4_L002_R1_001_val_1_val_1.fq.gz Analysis complete for Library_Geoduck_MG_2_S4_L002_R2_001_val_2_val_2.fq.gz Analysis complete for Library_Geoduck_MG_3_S1_L002_R1_001_val_1_val_1.fq.gz Analysis complete for Library_Geoduck_MG_3_S1_L002_R2_001_val_2_val_2.fq.gz Analysis complete for Library_Geoduck_MG_5_S6_L002_R1_001_val_1_val_1.fq.gz Analysis complete for Library_Geoduck_MG_5_S6_L002_R2_001_val_2_val_2.fq.gz Analysis complete for Library_Geoduck_MG_6_S5_L002_R1_001_val_1_val_1.fq.gz Analysis complete for Library_Geoduck_MG_6_S5_L002_R2_001_val_2_val_2.fq.gz Analysis complete for Library_Geoduck_MG_7_S2_L002_R1_001_val_1_val_1.fq.gz Analysis complete for Library_Geoduck_MG_7_S2_L002_R2_001_val_2_val_2.fq.gz -------------------- TAIL STDERR -------------------- Approx 80% complete for Library_Geoduck_MG_7_S2_L002_R2_001_val_2_val_2.fq.gz Approx 85% complete for Library_Geoduck_MG_7_S2_L002_R2_001_val_2_val_2.fq.gz Approx 90% complete for Library_Geoduck_MG_7_S2_L002_R2_001_val_2_val_2.fq.gz Approx 95% complete for Library_Geoduck_MG_7_S2_L002_R2_001_val_2_val_2.fq.gz Deleting both intermediate output files Library_Geoduck_MG_7_S2_L002_R1_001_val_1_trimmed.fq.gz and Library_Geoduck_MG_7_S2_L002_R2_001_val_2_trimmed.fq.gz ==================================================================================================== gzip: stdout: Broken pipe -------------------- -------------------- TRIMGALORE LISTING total 67G -rw-rw-r-- 1 sam sam 5.2K Dec 13 08:56 Library_Geoduck_MG_1_S3_L002_R1_001_val_1.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.7G Dec 13 10:32 Library_Geoduck_MG_1_S3_L002_R1_001_val_1_val_1.fq.gz -rw-rw-r-- 1 sam sam 6.2G Dec 13 10:32 Library_Geoduck_MG_1_S3_L002_R2_001_val_2_val_2.fq.gz -rw-rw-r-- 1 sam sam 5.3K Dec 13 10:32 Library_Geoduck_MG_1_S3_L002_R2_001_val_2.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.2K Dec 13 11:56 Library_Geoduck_MG_2_S4_L002_R1_001_val_1.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.4G Dec 13 13:29 Library_Geoduck_MG_2_S4_L002_R1_001_val_1_val_1.fq.gz -rw-rw-r-- 1 sam sam 6.0G Dec 13 13:29 Library_Geoduck_MG_2_S4_L002_R2_001_val_2_val_2.fq.gz -rw-rw-r-- 1 sam sam 5.2K Dec 13 13:29 Library_Geoduck_MG_2_S4_L002_R2_001_val_2.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.2K Dec 13 14:50 Library_Geoduck_MG_3_S1_L002_R1_001_val_1.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.3G Dec 13 16:20 Library_Geoduck_MG_3_S1_L002_R1_001_val_1_val_1.fq.gz -rw-rw-r-- 1 sam sam 5.7G Dec 13 16:20 Library_Geoduck_MG_3_S1_L002_R2_001_val_2_val_2.fq.gz -rw-rw-r-- 1 sam sam 5.2K Dec 13 16:20 Library_Geoduck_MG_3_S1_L002_R2_001_val_2.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.2K Dec 13 17:38 Library_Geoduck_MG_5_S6_L002_R1_001_val_1.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 4.8G Dec 13 19:00 Library_Geoduck_MG_5_S6_L002_R1_001_val_1_val_1.fq.gz -rw-rw-r-- 1 sam sam 5.3G Dec 13 19:00 Library_Geoduck_MG_5_S6_L002_R2_001_val_2_val_2.fq.gz -rw-rw-r-- 1 sam sam 5.1K Dec 13 19:00 Library_Geoduck_MG_5_S6_L002_R2_001_val_2.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.2K Dec 13 20:15 Library_Geoduck_MG_6_S5_L002_R1_001_val_1.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 4.9G Dec 13 21:38 Library_Geoduck_MG_6_S5_L002_R1_001_val_1_val_1.fq.gz -rw-rw-r-- 1 sam sam 5.4G Dec 13 21:38 Library_Geoduck_MG_6_S5_L002_R2_001_val_2_val_2.fq.gz -rw-rw-r-- 1 sam sam 5.1K Dec 13 21:38 Library_Geoduck_MG_6_S5_L002_R2_001_val_2.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.2K Dec 13 23:01 Library_Geoduck_MG_7_S2_L002_R1_001_val_1.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 5.8G Dec 14 00:41 Library_Geoduck_MG_7_S2_L002_R1_001_val_1_val_1.fq.gz -rw-rw-r-- 1 sam sam 6.4G Dec 14 00:41 Library_Geoduck_MG_7_S2_L002_R2_001_val_2_val_2.fq.gz -rw-rw-r-- 1 sam sam 5.1K Dec 14 00:41 Library_Geoduck_MG_7_S2_L002_R2_001_val_2.fq.gz_trimming_report.txt -rw-rw-r-- 1 sam sam 917 Dec 14 01:21 20181211_trimgalore.stdout.log -rw-rw-r-- 1 sam sam 97K Dec 14 01:21 20181211_trimgalore.stderr.log drwxrwxr-x 3 sam sam 4.0K Dec 14 01:22 20181211_metagenomics_trimmed_fastqc -------------------- -------------------- TRIMMED FASTQC LISTING total 7.5M -rw-rw-r-- 1 sam sam 290K Dec 13 10:51 Library_Geoduck_MG_1_S3_L002_R1_001_val_1_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 239K Dec 13 10:51 Library_Geoduck_MG_1_S3_L002_R1_001_val_1_val_1_fastqc.html -rw-rw-r-- 1 sam sam 297K Dec 13 11:10 Library_Geoduck_MG_1_S3_L002_R2_001_val_2_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 242K Dec 13 11:10 Library_Geoduck_MG_1_S3_L002_R2_001_val_2_val_2_fastqc.html -rw-rw-r-- 1 sam sam 292K Dec 13 13:47 Library_Geoduck_MG_2_S4_L002_R1_001_val_1_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 239K Dec 13 13:47 Library_Geoduck_MG_2_S4_L002_R1_001_val_1_val_1_fastqc.html -rw-rw-r-- 1 sam sam 293K Dec 13 14:06 Library_Geoduck_MG_2_S4_L002_R2_001_val_2_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 241K Dec 13 14:06 Library_Geoduck_MG_2_S4_L002_R2_001_val_2_val_2_fastqc.html -rw-rw-r-- 1 sam sam 290K Dec 13 16:38 Library_Geoduck_MG_3_S1_L002_R1_001_val_1_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 238K Dec 13 16:38 Library_Geoduck_MG_3_S1_L002_R1_001_val_1_val_1_fastqc.html -rw-rw-r-- 1 sam sam 289K Dec 13 16:57 Library_Geoduck_MG_3_S1_L002_R2_001_val_2_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 239K Dec 13 16:57 Library_Geoduck_MG_3_S1_L002_R2_001_val_2_val_2_fastqc.html -rw-rw-r-- 1 sam sam 292K Dec 13 19:16 Library_Geoduck_MG_5_S6_L002_R1_001_val_1_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 241K Dec 13 19:16 Library_Geoduck_MG_5_S6_L002_R1_001_val_1_val_1_fastqc.html -rw-rw-r-- 1 sam sam 294K Dec 13 19:33 Library_Geoduck_MG_5_S6_L002_R2_001_val_2_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 242K Dec 13 19:33 Library_Geoduck_MG_5_S6_L002_R2_001_val_2_val_2_fastqc.html -rw-rw-r-- 1 sam sam 291K Dec 13 21:56 Library_Geoduck_MG_6_S5_L002_R1_001_val_1_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 240K Dec 13 21:56 Library_Geoduck_MG_6_S5_L002_R1_001_val_1_val_1_fastqc.html -rw-rw-r-- 1 sam sam 294K Dec 13 22:12 Library_Geoduck_MG_6_S5_L002_R2_001_val_2_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 242K Dec 13 22:12 Library_Geoduck_MG_6_S5_L002_R2_001_val_2_val_2_fastqc.html -rw-rw-r-- 1 sam sam 290K Dec 14 01:01 Library_Geoduck_MG_7_S2_L002_R1_001_val_1_val_1_fastqc.zip -rw-rw-r-- 1 sam sam 239K Dec 14 01:01 Library_Geoduck_MG_7_S2_L002_R1_001_val_1_val_1_fastqc.html -rw-rw-r-- 1 sam sam 296K Dec 14 01:21 Library_Geoduck_MG_7_S2_L002_R2_001_val_2_val_2_fastqc.zip -rw-rw-r-- 1 sam sam 243K Dec 14 01:21 Library_Geoduck_MG_7_S2_L002_R2_001_val_2_val_2_fastqc.html -rw-rw-r-- 1 sam sam 1.2M Dec 14 01:22 multiqc_report.html drwxrwxr-x 2 sam sam 4.0K Dec 14 01:22 multiqc_data --------------------
real 1033m27.302s user 1884m3.520s sys 34m41.632s [INFO ] multiqc : This is MultiQC v1.6.dev0 [INFO ] multiqc : Template : default [INFO ] multiqc : Searching '.' [INFO ] fastqc : Found 12 reports [INFO ] multiqc : Compressing plot data [INFO ] multiqc : Report : multiqc_report.html [INFO ] multiqc : Data : multiqc_data [INFO ] multiqc : MultiQC complete
%%bash
cd /home/sam/analyses
time \
rsync \
--archive \
--relative \
./20181211_metagenomics_fastqc_trimgalore \
gannet:/volume1/web/Atumefaciens
sed '/^Subject:/ s/ / trimgalore file transfer JOB COMPLETE/' ~/.default-subject.mail | msmtp "$EMAIL"
real 24m7.851s user 20m47.296s sys 4m54.044s
%%bash
mkdir /home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_03
mkdir /home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_03/20181211_metagenomics_trimmed_fastqc
trimmed_fastq_dir="/home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_02"
trim_out_dir="/home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_03"
fastqc_out_dir="/home/sam/analyses/20181211_metagenomics_fastqc_trimgalore/20181211_metagenomics_trimgalore_03/20181211_metagenomics_trimmed_fastqc"
cd ${trimmed_fastq_dir}
# List all gzip files and then replace newline with space
FASTQ_LIST="$(ls -1 *.gz| tr '\n' ' ')"
# Run TrimGalore! with FastQC
# Trim first 10bp from each read.
time \
/home/shared/TrimGalore-0.4.5/trim_galore \
--paired \
--output_dir ${trim_out_dir} \
--clip_R1 10 \
--clip_R2 10 \
--fastqc_args \
"--outdir ${fastqc_out_dir} \
--threads 16" \
${FASTQ_LIST} \
> ${trim_out_dir}/20181211_trimgalore.stdout.log \
2> ${trim_out_dir}/20181211_trimgalore.stderr.log
# Run MultiQC
cd ${fastqc_out_dir}
multiqc .
Searching 24 files..
real 951m10.990s user 1767m39.528s sys 32m45.888s [INFO ] multiqc : This is MultiQC v1.6.dev0 [INFO ] multiqc : Template : default [INFO ] multiqc : Searching '.' [INFO ] fastqc : Found 12 reports [INFO ] multiqc : Compressing plot data [INFO ] multiqc : Report : multiqc_report.html [INFO ] multiqc : Data : multiqc_data [INFO ] multiqc : MultiQC complete
%%bash
cd /home/sam/analyses
time \
rsync \
--archive \
--relative \
./20181211_metagenomics_fastqc_trimgalore \
gannet:/volume1/web/Atumefaciens
sed '/^Subject:/ s/ / trimgalore file transfer JOB COMPLETE/' ~/.default-subject.mail | msmtp "$EMAIL"
real 14m53.006s user 9m40.336s sys 2m6.248s