More filtering, reuse find results

run.sh

@@ -1,41 +1,127 @@
 #!/usr/bin/env bash
 
 OUTPUT_DIR="out"
-
 mkdir -p "$OUTPUT_DIR"
 
+# TODO: all paths in cgi-bin should be considered a cgi-script, independent of file extension
+
+echo "Starting webdoc analysis..."
+
 if [[ ! -f "$OUTPUT_DIR/cgi-paths.txt" ]]; then
-  find \
+  echo "Searching for CGI scripts..."
-    /home/pvv/?/*/web-docs \
+
-    -type f \
+  EXCLUDE_LOCATIONS=(
-    \( \
+    "*/.bundle/*"
-         -name '*.cgi' \
+    "*/.bzr/*"
-      -o -name '*.php' \
+    "*/.cache/*"
-      -o -name '*.php3' \
+    "*/.cargo/*"
-      -o -name '*.pm' \
+    "*/.git/*"
-      -o -name '*.pl' \
+    "*/.hg/*"
-      -o -name '*.sh' \
+    "*/.npm/*"
-      -o -name '*.bash' \
+    "*/.nvm/*"
-      -o -name '*.phtml' \
+    "*/.rvm/*"
-      -o -name '*.shtml' \
+    "*/.svn/*"
-      -o -name '*.lisp' \
+    "*/.tox/*"
-      -o -name '*.cl' \
+    "*/__pycache__/*"
-    \) \
+    "*/node_modules/*"
-    2>/dev/null \
+    "*/vendor/*"
-    | tee "$OUTPUT_DIR/cgi-paths.txt"
+  )
+
+  FILE_NAME_PATTERNS=(
+    "*.cgi"
+    "*.php"
+    "*.php[0-9]*"
+    "*.pm"
+    "*.pl"
+    "*.sh"
+    "*.bash"
+    "*.phtml"
+    "*.shtml"
+    "*.lisp"
+    "*.cl"
+  )
+
+  FILE_NAME_ANTI_PATTERNS=(
+    "*.so"
+    "#*"
+    "*~"
+  )
+
+
+  FIND_ARGS=()
+  for d in /home/pvv/?/*/web-docs; do
+    if [[ -d "$d" ]]; then
+      FIND_ARGS+=("$d")
+    fi
+  done
+
+  if [[ ${#EXCLUDE_LOCATIONS[@]} -gt 0 ]]; then
+    FIND_ARGS+=( \( )
+    for ((i=0; i<${#EXCLUDE_LOCATIONS[@]}; i++)); do
+      FIND_ARGS+=(-path "${EXCLUDE_LOCATIONS[i]}")
+      if [[ $i -lt $(( ${#EXCLUDE_LOCATIONS[@]} - 1 )) ]]; then
+        FIND_ARGS+=(-o)
+      fi
+    done
+    FIND_ARGS+=( \) -prune -o )
+  fi
+
+  FIND_ARGS+=(
+    "-type" "f"
+    "-executable"
+  )
+
+  FIND_ARGS+=(\( )
+  for ((i=0; i<${#FILE_NAME_PATTERNS[@]}; i++)); do
+    pattern="${FILE_NAME_PATTERNS[i]}"
+    if [[ $i -gt 0 ]]; then
+      FIND_ARGS+=(-o)
+    fi
+    FIND_ARGS+=(-name "$pattern")
+  done
+  FIND_ARGS+=(\) )
+
+  for anti_pattern in "${FILE_NAME_ANTI_PATTERNS[@]}"; do
+    FIND_ARGS+=(-not -name "$anti_pattern")
+  done
+
+  FIND_ARGS+=("-print")
+
+  echo "find \\"
+  for arg in "${FIND_ARGS[@]}"; do
+    echo "  '$arg' \\"
+  done
+  echo ""
+
+  find "${FIND_ARGS[@]}" 2>/dev/null | tee "$OUTPUT_DIR/cgi-paths.txt"
+else
+  echo "'$OUTPUT_DIR/cgi-paths.txt' already exists, reusing..."
 fi
 
-if [[ ! -f out/perl-libs.txt ]]; then
+##########################
-  find \
+# ANALYZE PERL LIBRARIES #
-    /home/pvv/?/*/web-docs \
+##########################
-    -type f \
+
-    \( \
+if [[ ! -f "$OUTPUT_DIR/perl-libs.txt" ]]; then
-         -name '*.pm' \
+  echo "Analyzing Perl libraries..."
-      -o -name '*.pl' \
+
-    \) \
+  : > "$OUTPUT_DIR/perl-libs.txt"
-    -exec rg '^use ([^;]+);' {} -N -o -r '$1' \; \
+  mapfile -t ALL_CGI_PATHS < "$OUTPUT_DIR/cgi-paths.txt"
-    2>/dev/null \
+
-    | tee "$OUTPUT_DIR/perl-libs.txt"
+  PERL_PATHS=()
+  for p in "${ALL_CGI_PATHS[@]}"; do
+    case "$p" in
+      *.pm|*.pl)
+        PERL_PATHS+=("$p")
+        ;;
+    esac
+  done
+
+  for p in "${PERL_PATHS[@]}"; do
+    rg '^use ([^;]+);' --no-messages -N -o -r '$1' "$p" 2>/dev/null >> "$OUTPUT_DIR/perl-libs.txt" || true
+  done
+else
+  echo "'$OUTPUT_DIR/perl-libs.txt' already exists, reusing..."
 fi
 
 cat "$OUTPUT_DIR/perl-libs.txt" | sort > "$OUTPUT_DIR/perl-libs-tmp1.txt"
@@ -52,66 +138,67 @@ comm -2 -3 "$OUTPUT_DIR/perl-libs-tmp2.txt" <(cat PERL_STANDARD_MODULES.txt | so
 # Remove pragmas
 readarray -t PERL_PRAGMAS < PERL_PRAGMAS.txt
 remove_pragmas_regex=$(printf '|^%s' "${PERL_PRAGMAS[@]}")
-remove_pragmas_regex="${remove_pragmas_regex:1}"  # remove leading '|
+remove_pragmas_regex="${remove_pragmas_regex:1}"  # remove leading '|'
 sed -E "/${remove_pragmas_regex}/d" "$OUTPUT_DIR/perl-libs-tmp3.txt" > "$OUTPUT_DIR/perl-libs-filtered.txt"
 
 cat "$OUTPUT_DIR/perl-libs-filtered.txt" | uniq -c | sort -gr > "$OUTPUT_DIR/perl-libs-overview.txt"
 
+#########################
+# ANALYZE PHP LIBRARIES #
+#########################
+
 if [[ ! -f "$OUTPUT_DIR/php-libs.txt" ]]; then
-  find \
+  echo "Analyzing PHP libraries..."
-    /home/pvv/?/*/web-docs \
+
-    -type f \
+  : > "$OUTPUT_DIR/php-libs.txt"
-    \( \
+  mapfile -t ALL_CGI_PATHS < "$OUTPUT_DIR/cgi-paths.txt"
-         -name '*.php' \
+
-      -o -name '*.php3' \
+  PHP_PATHS=()
-      -o -name '*.phtml' \
+  for p in "${ALL_CGI_PATHS[@]}"; do
-    \) \
+    case "$p" in
-    -exec rg '^use ([^;]+);' {} -N -o -r '$1' \; \
+      *.php[0-9]*|*.phtml)
-    2>/dev/null \
+        PHP_PATHS+=("$p")
-    | tee "$OUTPUT_DIR/php-libs.txt"
+        ;;
+    esac
+  done
+
+  for p in "${PHP_PATHS[@]}"; do
+    rg '^use ([^;]+);' --no-messages -N -o -r '$1' "$p" 2>/dev/null >> "$OUTPUT_DIR/php-libs.txt" || true
+  done
+else
+  echo "'$OUTPUT_DIR/php-libs.txt' already exists, reusing..."
 fi
 
 cat "$OUTPUT_DIR/php-libs.txt" | sort | uniq -c | sort -gr > "$OUTPUT_DIR/php-libs-overview.txt"
 
+########################
+# ANALYZE CGI PROGRAMS #
+########################
+
 if [[ ! -f "$OUTPUT_DIR/cgi-progs.txt" ]]; then
+  echo "Analyzing CGI programs..."
+
+  : > "$OUTPUT_DIR/cgi-progs.txt"
+  mapfile -t ALL_CGI_PATHS < "$OUTPUT_DIR/cgi-paths.txt"
+
   readarray -t BIN_LOCATIONS < BIN_LOCATIONS.txt
   bin_locations_regex=$(printf '|%s' "${BIN_LOCATIONS[@]}")
   bin_locations_regex="${bin_locations_regex:1}"  # remove leading '|'
   bin_locations_regex="(?:${bin_locations_regex})"
 
-  find \
+  for p in "${ALL_CGI_PATHS[@]}"; do
-    /home/pvv/?/*/web-docs \
+    rg "$bin_locations_regex/(?:env\\s*)?(\\w+(?:/\\w+)*)" --no-messages -N -o -r '$1' "$p" 2>/dev/null >> "$OUTPUT_DIR/cgi-progs.txt" || true
-    -type f \
+  done
-    \( \
+
-         -name '*.cgi' \
+  # TODO: extract non-absolute binary references from calls that invoke subprocesses in the various languages
-      -o -name '*.php' \
+else
-      -o -name '*.php3' \
+  echo "'$OUTPUT_DIR/cgi-progs.txt' already exists, reusing..."
-      -o -name '*.pm' \
-      -o -name '*.pl' \
-      -o -name '*.sh' \
-      -o -name '*.bash' \
-      -o -name '*.phtml' \
-      -o -name '*.shtml' \
-      -o -name '*.lisp' \
-      -o -name '*.cl' \
-    \) \
-    -exec rg "$bin_locations_regex/(?:env\s*)?(\w+(?:/\w+)*)" {} -N -o -r '$1' \; \
-    2>/dev/null \
-    | tee "$OUTPUT_DIR/cgi-progs.txt"
 fi
 
 cat "$OUTPUT_DIR/cgi-progs.txt" | sort | uniq -c | sort -gr > "$OUTPUT_DIR/cgi-progs-overview.txt"
 
+##########################
+# ANALYZE LISP LIBRARIES #
+##########################
+
 # TODO find lisp libraries
-
-# TODO: look through all files in ~/web-docs/cgi-bin (last one case insensitive)
-
-# TODO: generally filter out files ending with ~ or starting with #
-
-# TODO: generally filter out .git, RCS, and other such files
-
-# TODO: generally filter out executables, drop executable .so files
-
-# TODO: look for all digits behind '*.php' (i.e. asdf.php4 and asdf.php5)
-
-# TODO: extract non-absolute binary references from calls that invoke subprocesses in the various languages