commit ef397bd473c0b5671bd5a23d33bee1115b0803bb Author: Øystein Kristoffer Tveit Date: Wed Feb 4 09:26:58 2026 +0100 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1fcb152 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +out diff --git a/PERL_PRAGMAS.txt b/PERL_PRAGMAS.txt new file mode 100644 index 0000000..f645e34 --- /dev/null +++ b/PERL_PRAGMAS.txt @@ -0,0 +1,52 @@ +attributes +autodie +autodie::exception +autodie::exception::system +autodie::hints +autodie::skip +autouse +base +bigfloat +bigint +bignum +bigrat +blib +builtin +bytes +charnames +constant +deprecate +diagnostics +encoding +encoding::warnings +experimental +feature +fields +filetest +if +integer +less +lib +locale +mro +ok +open +ops +overload +overloading +parent +re +sigtrap +sort +source::encoding +stable +strict +subs +threads +threads::shared +utf8 +vars +version +vmsish +warnings +warnings::register diff --git a/PERL_STANDARD_MODULES.txt b/PERL_STANDARD_MODULES.txt new file mode 100644 index 0000000..ae99f3d --- /dev/null +++ b/PERL_STANDARD_MODULES.txt @@ -0,0 +1,666 @@ +Amiga::ARexx +Amiga::Exec +AnyDBM_File +App::Cpan +App::Prove +App::Prove::State +App::Prove::State::Result +App::Prove::State::Result::Test +Archive::Tar +Archive::Tar::File +Attribute::Handlers +AutoLoader +AutoSplit +B +B::Concise +B::Deparse +B::Op_private +B::Showlex +B::Terse +B::Xref +Benchmark +IO::Socket::IP +Socket +Test2::Tools::Refcount +CORE +CPAN +CPAN::API::HOWTO +CPAN::Debug +CPAN::Distroprefs +CPAN::FirstTime +CPAN::HandleConfig +CPAN::Kwalify +CPAN::Meta +CPAN::Meta::Converter +CPAN::Meta::Feature +CPAN::Meta::History +CPAN::Meta::History::Meta_1_0 +CPAN::Meta::History::Meta_1_1 +CPAN::Meta::History::Meta_1_2 +CPAN::Meta::History::Meta_1_3 +CPAN::Meta::History::Meta_1_4 +CPAN::Meta::Merge +CPAN::Meta::Prereqs +CPAN::Meta::Requirements +CPAN::Meta::Requirements::Range +CPAN::Meta::Spec +CPAN::Meta::Validator +CPAN::Meta::YAML +CPAN::Nox +CPAN::Plugin +CPAN::Plugin::Specfile +CPAN::Queue +CPAN::Tarzip +CPAN::Version +Carp +Class::Struct +Compress::Raw::Bzip2 +Compress::Raw::Zlib +Compress::Zlib +Config +Config::Extensions +Config::Perl::V +Cwd +DB +DBM_Filter +DBM_Filter::compress +DBM_Filter::encode +DBM_Filter::int32 +DBM_Filter::null +DBM_Filter::utf8 +DB_File +Data::Dumper +Devel::PPPort +Devel::Peek +Devel::SelfStubber +Digest +Digest::MD5 +Digest::SHA +Digest::base +Digest::file +DirHandle +Dumpvalue +DynaLoader +Encode +Encode::Alias +Encode::Byte +Encode::CJKConstants +Encode::CN +Encode::CN::HZ +Encode::Config +Encode::EBCDIC +Encode::Encoder +Encode::Encoding +Encode::GSM0338 +Encode::Guess +Encode::JP +Encode::JP::H2Z +Encode::JP::JIS7 +Encode::KR +Encode::KR::2022_KR +Encode::MIME::Header +Encode::MIME::Name +Encode::PerlIO +Encode::Supported +Encode::Symbol +Encode::TW +Encode::Unicode +Encode::Unicode::UTF7 +English +Env +Errno +Exporter +Exporter::Heavy +ExtUtils::CBuilder +ExtUtils::CBuilder::Platform::Windows +ExtUtils::Command +ExtUtils::Command::MM +ExtUtils::Constant +ExtUtils::Constant::Base +ExtUtils::Constant::Utils +ExtUtils::Constant::XS +ExtUtils::Embed +ExtUtils::Install +ExtUtils::Installed +ExtUtils::Liblist +ExtUtils::MM +ExtUtils::MM_AIX +ExtUtils::MM_Any +ExtUtils::MM_BeOS +ExtUtils::MM_Cygwin +ExtUtils::MM_DOS +ExtUtils::MM_Darwin +ExtUtils::MM_MacOS +ExtUtils::MM_NW5 +ExtUtils::MM_OS2 +ExtUtils::MM_OS390 +ExtUtils::MM_QNX +ExtUtils::MM_UWIN +ExtUtils::MM_Unix +ExtUtils::MM_VMS +ExtUtils::MM_VOS +ExtUtils::MM_Win32 +ExtUtils::MM_Win95 +ExtUtils::MY +ExtUtils::MakeMaker +ExtUtils::MakeMaker::Config +ExtUtils::MakeMaker::FAQ +ExtUtils::MakeMaker::Locale +ExtUtils::MakeMaker::Tutorial +ExtUtils::Manifest +ExtUtils::Miniperl +ExtUtils::Mkbootstrap +ExtUtils::Mksymlists +ExtUtils::PL2Bat +ExtUtils::Packlist +ExtUtils::ParseXS +ExtUtils::ParseXS::Constants +ExtUtils::ParseXS::Eval +ExtUtils::ParseXS::Node +ExtUtils::ParseXS::Utilities +ExtUtils::Typemaps +ExtUtils::Typemaps::Cmd +ExtUtils::Typemaps::InputMap +ExtUtils::Typemaps::OutputMap +ExtUtils::Typemaps::Type +ExtUtils::XSSymSet +ExtUtils::testlib +Fatal +Fcntl +File::Basename +File::Compare +File::Copy +File::DosGlob +File::Fetch +File::Find +File::Glob +File::GlobMapper +File::Path +File::Spec +File::Spec::AmigaOS +File::Spec::Cygwin +File::Spec::Epoc +File::Spec::Functions +File::Spec::Mac +File::Spec::OS2 +File::Spec::Unix +File::Spec::VMS +File::Spec::Win32 +File::Temp +File::stat +FileCache +FileHandle +Filter::Simple +Filter::Util::Call +FindBin +GDBM_File +Getopt::Long +Getopt::Long::Parser +Getopt::Std +HTTP::Tiny +Hash::Util +Hash::Util::FieldHash +I18N::Collate +I18N::LangTags +I18N::LangTags::Detect +I18N::LangTags::List +I18N::Langinfo +IO +IO::Compress +IO::Compress::Base +IO::Compress::Bzip2 +IO::Compress::Deflate +IO::Compress::FAQ +IO::Compress::Gzip +IO::Compress::RawDeflate +IO::Compress::Zip +IO::Dir +IO::File +IO::Handle +IO::Pipe +IO::Poll +IO::Seekable +IO::Select +IO::Socket +IO::Socket::INET +IO::Socket::UNIX +IO::Uncompress::AnyInflate +IO::Uncompress::AnyUncompress +IO::Uncompress::Base +IO::Uncompress::Bunzip2 +IO::Uncompress::Gunzip +IO::Uncompress::Inflate +IO::Uncompress::RawInflate +IO::Uncompress::Unzip +IO::Zlib +IPC::Cmd +IPC::Msg +IPC::Open2 +IPC::Open3 +IPC::Semaphore +IPC::SharedMem +IPC::SysV +Internals +JSON::PP +JSON::PP::Boolean +List::Util +List::Util::XS +Locale::Maketext +Locale::Maketext::Cookbook +Locale::Maketext::Guts +Locale::Maketext::GutsLoader +Locale::Maketext::Simple +Locale::Maketext::TPJ13 +MIME::Base64 +MIME::QuotedPrint +Math::BigFloat +Math::BigInt +Math::BigInt::Calc +Math::BigInt::FastCalc +Math::BigInt::Lib +Math::BigRat +Math::Complex +Math::Trig +Memoize +Memoize::AnyDBM_File +Memoize::Expire +Memoize::NDBM_File +Memoize::SDBM_File +Memoize::Storable +Module::CoreList +Module::CoreList::Utils +Module::Load +Module::Load::Conditional +Module::Loaded +Module::Metadata +NDBM_File +NEXT +Net::Cmd +Net::Config +Net::Domain +Net::FTP +Net::FTP::dataconn +Net::NNTP +Net::Netrc +Net::POP3 +Net::Ping +Net::SMTP +Net::Time +Net::hostent +Net::libnetFAQ +Net::netent +Net::protoent +Net::servent +O +ODBM_File +Opcode +POSIX +Params::Check +Parse::CPAN::Meta +Perl::OSType +PerlIO +PerlIO::encoding +PerlIO::mmap +PerlIO::scalar +PerlIO::via +PerlIO::via::QuotedPrint +Pod::Checker +Pod::Escapes +Pod::Functions +Pod::Html +Pod::Html::Util +Pod::Man +Pod::ParseLink +Pod::Perldoc +Pod::Perldoc::BaseTo +Pod::Perldoc::GetOptsOO +Pod::Perldoc::ToANSI +Pod::Perldoc::ToChecker +Pod::Perldoc::ToMan +Pod::Perldoc::ToNroff +Pod::Perldoc::ToPod +Pod::Perldoc::ToRtf +Pod::Perldoc::ToTerm +Pod::Perldoc::ToText +Pod::Perldoc::ToTk +Pod::Perldoc::ToXml +Pod::Simple +Pod::Simple::Checker +Pod::Simple::Debug +Pod::Simple::DumpAsText +Pod::Simple::DumpAsXML +Pod::Simple::HTML +Pod::Simple::HTMLBatch +Pod::Simple::JustPod +Pod::Simple::LinkSection +Pod::Simple::Methody +Pod::Simple::PullParser +Pod::Simple::PullParserEndToken +Pod::Simple::PullParserStartToken +Pod::Simple::PullParserTextToken +Pod::Simple::PullParserToken +Pod::Simple::RTF +Pod::Simple::Search +Pod::Simple::SimpleTree +Pod::Simple::Subclassing +Pod::Simple::Text +Pod::Simple::TextContent +Pod::Simple::XHTML +Pod::Simple::XMLOutStream +Pod::Text +Pod::Text::Color +Pod::Text::Overstrike +Pod::Text::Termcap +Pod::Usage +SDBM_File +Safe +Scalar::List::Utils +Scalar::Util +Search::Dict +SelectSaver +SelfLoader +Storable +Sub::Util +Symbol +Sys::Hostname +Sys::Syslog +Sys::Syslog::Win32 +TAP::Base +TAP::Formatter::Base +TAP::Formatter::Color +TAP::Formatter::Console +TAP::Formatter::Console::ParallelSession +TAP::Formatter::Console::Session +TAP::Formatter::File +TAP::Formatter::File::Session +TAP::Formatter::Session +TAP::Harness +TAP::Harness::Env +TAP::Object +TAP::Parser +TAP::Parser::Aggregator +TAP::Parser::Grammar +TAP::Parser::Iterator +TAP::Parser::Iterator::Array +TAP::Parser::Iterator::Process +TAP::Parser::Iterator::Stream +TAP::Parser::IteratorFactory +TAP::Parser::Multiplexer +TAP::Parser::Result +TAP::Parser::Result::Bailout +TAP::Parser::Result::Comment +TAP::Parser::Result::Plan +TAP::Parser::Result::Pragma +TAP::Parser::Result::Test +TAP::Parser::Result::Unknown +TAP::Parser::Result::Version +TAP::Parser::Result::YAML +TAP::Parser::ResultFactory +TAP::Parser::Scheduler +TAP::Parser::Scheduler::Job +TAP::Parser::Scheduler::Spinner +TAP::Parser::Source +TAP::Parser::SourceHandler +TAP::Parser::SourceHandler::Executable +TAP::Parser::SourceHandler::File +TAP::Parser::SourceHandler::Handle +TAP::Parser::SourceHandler::Perl +TAP::Parser::SourceHandler::RawTAP +TAP::Parser::YAMLish::Reader +TAP::Parser::YAMLish::Writer +Term::ANSIColor +Term::Cap +Term::Complete +Term::ReadLine +Term::Table +Term::Table::Cell +Term::Table::CellStack +Term::Table::HashBase +Term::Table::LineBreak +Term::Table::Util +Test +Test2 +Test2::API +Test2::API::Breakage +Test2::API::Context +Test2::API::Instance +Test2::API::InterceptResult +Test2::API::InterceptResult::Event +Test2::API::InterceptResult::Hub +Test2::API::InterceptResult::Squasher +Test2::API::Stack +Test2::AsyncSubtest +Test2::AsyncSubtest::Event::Attach +Test2::AsyncSubtest::Event::Detach +Test2::AsyncSubtest::Hub +Test2::Bundle +Test2::Bundle::Extended +Test2::Bundle::More +Test2::Bundle::Simple +Test2::Compare +Test2::Compare::Array +Test2::Compare::Bag +Test2::Compare::Base +Test2::Compare::Bool +Test2::Compare::Custom +Test2::Compare::DeepRef +Test2::Compare::Delta +Test2::Compare::Event +Test2::Compare::EventMeta +Test2::Compare::Float +Test2::Compare::Hash +Test2::Compare::Isa +Test2::Compare::Meta +Test2::Compare::Negatable +Test2::Compare::Number +Test2::Compare::Object +Test2::Compare::OrderedSubset +Test2::Compare::Pattern +Test2::Compare::Ref +Test2::Compare::Regex +Test2::Compare::Scalar +Test2::Compare::Set +Test2::Compare::String +Test2::Compare::Undef +Test2::Compare::Wildcard +Test2::Env +Test2::Event +Test2::Event::Bail +Test2::Event::Diag +Test2::Event::Encoding +Test2::Event::Exception +Test2::Event::Fail +Test2::Event::Generic +Test2::Event::Note +Test2::Event::Ok +Test2::Event::Pass +Test2::Event::Plan +Test2::Event::Skip +Test2::Event::Subtest +Test2::Event::TAP::Version +Test2::Event::V2 +Test2::Event::Waiting +Test2::EventFacet +Test2::EventFacet::About +Test2::EventFacet::Amnesty +Test2::EventFacet::Assert +Test2::EventFacet::Control +Test2::EventFacet::Error +Test2::EventFacet::Hub +Test2::EventFacet::Info +Test2::EventFacet::Info::Table +Test2::EventFacet::Meta +Test2::EventFacet::Parent +Test2::EventFacet::Plan +Test2::EventFacet::Render +Test2::EventFacet::Trace +Test2::Formatter +Test2::Formatter::TAP +Test2::Hub +Test2::Hub::Interceptor +Test2::Hub::Interceptor::Terminator +Test2::Hub::Subtest +Test2::IPC +Test2::IPC::Driver +Test2::IPC::Driver::Files +Test2::Manual +Test2::Manual::Anatomy +Test2::Manual::Anatomy::API +Test2::Manual::Anatomy::Context +Test2::Manual::Anatomy::EndToEnd +Test2::Manual::Anatomy::Event +Test2::Manual::Anatomy::Hubs +Test2::Manual::Anatomy::IPC +Test2::Manual::Anatomy::Utilities +Test2::Manual::Concurrency +Test2::Manual::Contributing +Test2::Manual::Testing +Test2::Manual::Testing::Introduction +Test2::Manual::Testing::Migrating +Test2::Manual::Testing::Planning +Test2::Manual::Testing::Todo +Test2::Manual::Tooling +Test2::Manual::Tooling::FirstTool +Test2::Manual::Tooling::Formatter +Test2::Manual::Tooling::Nesting +Test2::Manual::Tooling::Plugin::TestExit +Test2::Manual::Tooling::Plugin::TestingDone +Test2::Manual::Tooling::Plugin::ToolCompletes +Test2::Manual::Tooling::Plugin::ToolStarts +Test2::Manual::Tooling::Subtest +Test2::Manual::Tooling::TestBuilder +Test2::Manual::Tooling::Testing +Test2::Mock +Test2::Plugin +Test2::Plugin::BailOnFail +Test2::Plugin::DieOnFail +Test2::Plugin::ExitSummary +Test2::Plugin::SRand +Test2::Plugin::Times +Test2::Plugin::UTF8 +Test2::Require +Test2::Require::AuthorTesting +Test2::Require::AutomatedTesting +Test2::Require::EnvVar +Test2::Require::ExtendedTesting +Test2::Require::Fork +Test2::Require::Module +Test2::Require::NonInteractiveTesting +Test2::Require::Perl +Test2::Require::RealFork +Test2::Require::ReleaseTesting +Test2::Require::Threads +Test2::Suite +Test2::Todo +Test2::Tools +Test2::Tools::AsyncSubtest +Test2::Tools::Basic +Test2::Tools::Class +Test2::Tools::ClassicCompare +Test2::Tools::Compare +Test2::Tools::Defer +Test2::Tools::Encoding +Test2::Tools::Event +Test2::Tools::Exception +Test2::Tools::Exports +Test2::Tools::GenTemp +Test2::Tools::Grab +Test2::Tools::Mock +Test2::Tools::Ref +Test2::Tools::Spec +Test2::Tools::Subtest +Test2::Tools::Target +Test2::Tools::Tester +Test2::Tools::Tiny +Test2::Tools::Warnings +Test2::Transition +Test2::Util +Test2::Util::ExternalMeta +Test2::Util::Facets2Legacy +Test2::Util::Grabber +Test2::Util::Guard +Test2::Util::HashBase +Test2::Util::Importer +Test2::Util::Ref +Test2::Util::Sig +Test2::Util::Stash +Test2::Util::Sub +Test2::Util::Table +Test2::Util::Table::LineBreak +Test2::Util::Times +Test2::Util::Trace +Test2::V0 +Test2::Workflow +Test2::Workflow::BlockBase +Test2::Workflow::Build +Test2::Workflow::Runner +Test2::Workflow::Task +Test2::Workflow::Task::Action +Test2::Workflow::Task::Group +Test::Builder +Test::Builder::Formatter +Test::Builder::Module +Test::Builder::Tester +Test::Builder::Tester::Color +Test::Builder::TodoDiag +Test::Harness +Test::Harness::Beyond +Test::More +Test::Simple +Test::Tester +Test::Tester::Capture +Test::Tester::CaptureRunner +Test::Tutorial +Test::use::ok +Text::Abbrev +Text::Balanced +Text::ParseWords +Text::Tabs +Text::Wrap +Thread +Thread::Queue +Thread::Semaphore +Tie::Array +Tie::File +Tie::Handle +Tie::Hash +Tie::Hash::NamedCapture +Tie::Memoize +Tie::RefHash +Tie::Scalar +Tie::StdHandle +Tie::SubstrHash +Time::HiRes +Time::Local +Time::Piece +Time::Seconds +Time::gmtime +Time::localtime +Time::tm +UNIVERSAL +Unicode::Collate +Unicode::Collate::CJK::Big5 +Unicode::Collate::CJK::GB2312 +Unicode::Collate::CJK::JISX0208 +Unicode::Collate::CJK::Korean +Unicode::Collate::CJK::Pinyin +Unicode::Collate::CJK::Stroke +Unicode::Collate::CJK::Zhuyin +Unicode::Collate::Locale +Unicode::Normalize +Unicode::UCD +User::grent +User::pwent +VMS::DCLsym +VMS::Filespec +VMS::Stdio +Win32 +Win32API::File +Win32CORE +XS::APItest +XS::Typemap +XSLoader +autodie::Scope::Guard +autodie::Scope::GuardStack +autodie::Util +version::Internals diff --git a/README.md b/README.md new file mode 100644 index 0000000..eb578e2 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# webdoc-analysis + +Static analysis of `~/web-doc` directories. Let's try to determine what kinds of software and libraries people use, and what we should support. diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..8efc9da --- /dev/null +++ b/run.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +OUTPUT_DIR="out" + +mkdir -p "$OUTPUT_DIR" + +if [[ ! -f "$OUTPUT_DIR/cgi-paths.txt" ]]; then + find \ + /home/pvv/?/*/web-docs \ + -type f \ + \( \ + -name '*.cgi' \ + -o -name '*.php' \ + -o -name '*.php3' \ + -o -name '*.pm' \ + -o -name '*.pl' \ + -o -name '*.sh' \ + -o -name '*.bash' \ + -o -name '*.phtml' \ + -o -name '*.shtml' \ + -o -name '*.lisp' \ + -o -name '*.cl' \ + \) \ + 2>/dev/null \ + | tee "$OUTPUT_DIR/cgi-paths.txt" +fi + +if [[ ! -f out/perl-libs.txt ]]; then + find \ + /home/pvv/?/*/web-docs \ + -type f \ + \( \ + -name '*.pm' \ + -o -name '*.pl' \ + \) \ + -exec rg '^use ([^;]+);' {} -N -o -r '$1' \; \ + 2>/dev/null \ + | tee "$OUTPUT_DIR/perl-libs.txt" +fi + +cat "$OUTPUT_DIR/perl-libs.txt" | sort > "$OUTPUT_DIR/perl-libs-sorted.txt" +comm -2 -3 "$OUTPUT_DIR/perl-libs-sorted.txt" PERL_STANDARD_MODULES.txt > "$OUTPUT_DIR/perl-libs-filtered.txt" +# TODO: remove pragmas +cat "$OUTPUT_DIR/perl-libs-filtered.txt" | uniq -c | sort -gr > "$OUTPUT_DIR/perl-libs-overview.txt" + +if [[ ! -f "$OUTPUT_DIR/php-libs.txt" ]]; then + find \ + /home/pvv/?/*/web-docs \ + -type f \ + \( \ + -name '*.php' \ + -o -name '*.php3' \ + -o -name '*.phtml' \ + \) \ + -exec rg '^use ([^;]+);' {} -N -o -r '$1' \; \ + 2>/dev/null \ + | tee "$OUTPUT_DIR/php-libs.txt" +fi + +cat "$OUTPUT_DIR/php-libs.txt" | sort | uniq -c | sort -gr > "$OUTPUT_DIR/php-libs-overview.txt" + +find \ + /home/pvv/?/*/web-docs \ + -type f \ + \( \ + -name '*.cgi' \ + -o -name '*.php' \ + -o -name '*.php3' \ + -o -name '*.pm' \ + -o -name '*.pl' \ + -o -name '*.sh' \ + -o -name '*.bash' \ + -o -name '*.phtml' \ + -o -name '*.shtml' \ + -o -name '*.lisp' \ + -o -name '*.cl' \ + \) \ + -exec rg '(?:/store/bin|/store/gnu/bin|/usr/bin|/usr/local/bin|/local/bin|/opt/bin|/opt/local/bin|/sbin|/usr/sbin|/usr/local/sbin|/opt/sbin|/var/qmail/bin|/usr/drwho/local/bin|/usr/local/mlog3/bin|/usr/local/flite/bin|/opt/flite/bin|/opt/csw/bin|/sw/bin|/usr/sfw/bin|/perl/bin|/usr/local/php/bin|/usr/X11R6/bin)/(?:env\s*)?(\w+(?:/\w+)*)' {} -N -o -r '$1' \; \ + 2>/dev/null \ + | tee "$OUTPUT_DIR/cgi-progs.txt" + +cat "$OUTPUT_DIR/cgi-progs.txt" | sort | uniq -c | sort -gr > "$OUTPUT_DIR/cgi-progs-overview.txt" + +# TODO find lisp libraries