config/pkgs/pdocs.nix

{ lib
, pkgs
}:

# TODO: pagefind

let
  python-builtins = [
    "builtins"
    "os"
    "array"
    "sys"
    "time"
    "traceback"
    "pathlib"
    "itertools"
    "functools"
    "unittest"
    "argparse"
    "asyncio"
    "textwrap"
    "collections"
    "configparser"
    "concurrent"
    "contextlib"
    "operator"
    "pickle"
    "copy"
    "ctypes"
    "pprint"
    "shlex"
    "re"
    "abc"
    "ast"
    "random"
    "shutil"
    "sqlite3"
    "subprocess"
    "statistics"
    "string"
    "tarfile"
    "typing"
    "uuid"
    "warnings"
    "wave"
    "dataclasses"
    "glob"
    "gzip"
    "inspect"
    "json"
    "base64"
    "zipfile"
  ];


  #python-packages = with pkgs.python3Packages; [ cached-property ];

  #python-packages = lib.pipe pkgs.python3Packages [
  #  builtins.attrValues
  #  (builtins.filter lib.isDerivation)
  #];
  /** /
  python-packages = with pkgs.python3Packages; [
    more-itertools
    altair
    pygal
    vispy
    seaborn
    bokeh
    plotly
    tabulate
    wavefile
    moderngl
    pydantic
    typer
    ptpython
    colorama
    pyjwt
    zipp
    aiofiles
    aafigure
    urllib3
    tesserocr
    trio
    starlette
    pyverilog
    nixpkgs
    wavedrom
    httpx
    pyquery
    mpv
    beautifulsoup4
    hid
    hidapi
    #sanic # broken build?
    paramiko
    pydub
    aiohttp
    papermill
    rtoml
    redis
    numpy
    #domeneshop
    munch
    migen
    amaranth
    click
    attrs
    graphviz
    baron
    redbaron
    fastapi
    pytest
    #pyglet # pyglet.com fails, windows only
    #pygame # pygame.movie fails on pdoc3, pdoc hangs
    plotly
    peewee
    parsel
    pandas
    #mutmut # moved to toplevel from python3Packages
    mlflow
    meshio
    #einops # depends on tensorflow, which is broken ATM
    aiodns
    json5
    seaborn
    matplotlib
    dash
    rarfile
    pyramid
    pygtail
    codecov
    nbconvert
    humanfriendly
    pendulum
    jsonpickle
    cachetools
    wrapt
    lxml
    chardet
    yarl
    frozenlist
    itsdangerous
    xmltodict
    cached-property
    toolz
    aioitertools
    coconut
    asyncpg
    aiopg
    libsass
    pytorch
    pytorch-lightning
    pillow
    trio
    tqdm
    rich
    pudb
    pony
    mido
    jedi
    h5py
    atom
    toml
    pyyaml
    jinja2
    requests
    h5py
    imageio
    pygments
    trimesh
    shapely
    #faiss
    #geomloss
    #mesh_to_sdf
    #pyrender
  ];
  /**/
  python-packages = with pkgs.python3Packages; [
    aiocurrencylayer
    aioitertools
    aiolifx-connection
    aiolifx-effects
    aiomisc
    aionotify
    aiorun
    aioshutil
    aiozeroconf
    alembic
    aliyun-python-sdk-dbfs
    allure-python-commons-test
    amply
    angr
    aniso8601
    anonip
    ansible
    ansicolor
    ansiwrap
    apptools
    aprslib
    aqipy-atmotech
    arc4
    argcomplete
    args
    arpeggio
    asgi-csrf
    asn1tools
    aspectlib
    astor
    async-lru
    asynccmd
    asyncio-throttle
    asynctest
    asysocks
    atom
    atomicwrites-homeassistant
    attrdict
    autopage
    autopep8
    avea
    avro3k
    awacs
    awswrangler
    azure-mgmt-nspkg
    b2sdk
    behave
    bitarray
    bitcoinrpc
    bitlist
    bluetooth-auto-recovery
    bnunicodenormalizer
    boschshcpy
    bottleneck
    brelpy
    bsddb3
    bson
    bunch
    cart
    casa-formats-io
    cftime
    chacha20poly1305
    cmigemo
    coapthon3
    cogapp
    coinmetrics-api-client
    commentjson
    cons
    contexttimer
    contourpy
    coreapi
    cppheaderparser
    dash-table
    dask-jobqueue
    decli
    deep-chainmap
    diceware
    diff-cover
    django-bootstrap4
    django-cache-url
    django-cacheops
    django-celery-results
    django-compressor
    django-picklefield
    django-reversion
    django-tables2
    djangorestframework-guardian2
    djmail
    doit-py
    dotmap
    drf-nested-routers
    dugong
    dunamai
    dvc-render
    entrance-with-router-features
    ephemeral-port-reserve
    et_xmlfile
    eth-hash
    eth-keys
    eve
    exdown
    exif
    face
    fastbencode
    fastcache
    fastentrypoints
    fe25519
    filetype
    fingerprints
    fire
    fixtures
    flake8-future-import
    flask-gravatar
    flask-swagger
    flask-swagger-ui
    fpdf
    fs
    ftputil
    funcparserlib
    funcy
    fuzzywuzzy
    gbinder-python
    gcovr
    generic
    geoip
    geojson
    ghrepo-stats
    gibberish-detector
    google-cloud-bigquery-logging
    google-cloud-dns
    gpaw
    graphql-server-core
    greeclimate
    gunicorn
    gvm-tools
    headerparser
    heapdict
    hijri-converter
    hledger-utils
    htmllaundry
    httpie
    httpx
    hyperlink
    imageio-ffmpeg
    imaplib2
    importlib-resources
    inotifyrecursive
    inquirer
    insteon-frontend-home-assistant
    intelhex
    interface-meta
    ipwhl
    irctokens
    isounidecode
    itemloaders
    iteration-utilities
    itsdangerous
    itypes
    jaeger-client
    javaproperties
    jax
    joblib
    json-rpc
    json-tricks
    jsonpatch
    junit-xml
    jupyter-cache
    jupyter-packaging
    jupyterlab-pygments
    jupyterlab_launcher
    jxmlease
    keyrings-cryptfile
    korean-lunar-calendar
    kubernetes
    language-data
    lazy
    lcov_cobertura
    ldap3
    ledger
    libais
    libarchive-c
    libarcus
    libgpuarray
    license-expression
    lightwave
    lima
    lit
    lockfile
    log-symbols
    luhn
    m3u8
    magic-wormhole
    mail-parser
    manhole
    markups
    marshmallow-oneofschema
    marshmallow-polyfield
    mastodon-py
    maxminddb
    mdurl
    mdutils
    meep
    mergedict
    merkletools
    mip
    mkdocs
    mkdocs-material-extensions
    msoffcrypto-tool
    multimethod
    multipart
    multiprocess
    mypy
    nanoid
    napalm
    napalm-hp-procurve
    nbdime
    nbformat
    nbval
    ndtypes
    neo4j
    nessclient
    netdata
    nose-randomly
    notebook-shim
    nsz
    nulltype
    ome-zarr
    onetimepass
    oocsi
    opsdroid_get_image_size
    opytimark
    oracledb
    pa-ringbuffer
    pad4pi
    papermill
    parsimonious
    parsley
    pcapy-ng
    pdoc
    phonopy
    pick
    picobox
    pipdate
    pkce
    pkgconfig
    pkginfo
    plantuml
    platformdirs
    plum-py
    plyer
    plyvel
    progressbar33
    prometheus-client
    promise
    prox-tv
    pulumi-command
    pure-cdb
    py-dmidecode
    py-multiaddr
    py-multibase
    py-nextbusnext
    py-zabbix
    pyaehw4a1
    pyatv
    pybullet
    pycangjie
    pycddl
    pycep-parser
    pydevccu
    pyftdi
    pyfume
    pygatt
    pygetwindow
    pyglet
    pygmars
    pyhcl
    pyheos
    pyinstrument
    pykdtree
    pylint-flask
    pymeeus
    pymetar
    pymodbus
    pymysensors
    pypdf
    pypdf3
    pyprind
    pyqtwebengine
    pyrainbird
    pyrmvtransport
    pyro5
    pyrogram
    pyrr
    pyscss
    pysdl2
    pysearpc
    pysensors
    pyside2
    pysmf
    pysmi
    pysml
    pysmt
    pysnmp-pyasn1
    pyspf
    pysvg-py3
    pysychonaut
    pytest-bdd
    pytest-catchlog
    pytest-django
    pytest-expect
    pytest-factoryboy
    pytest-flask
    pytest-isort
    pytest-relaxed
    pytest-snapshot
    pytest-socket
    pytest-virtualenv
    pytestcache
    python-baseconv
    python-bidi
    python-daemon
    python-decouple
    python-editor
    python-ipware
    python-ldap-test
    python-packer
    python-socketio
    python-status
    python-u2flib-server
    pytimeparse
    pytm
    pytzdata
    pyvisa
    pywemo
    pyworld
    pyxl3
    qtile
    reactivex
    rebulk
    reikna
    related
    repath
    repoze_lru
    requests-pkcs12
    requirements-parser
    result
    retrying
    rich-argparse-plus
    rivet
    rouge-score
    rtp
    rx
    safe
    sasmodels
    scikit-bio
    scikit-fmm
    seccomp
    securetar
    sendgrid
    sentence-transformers
    serialio
    setuptools-git
    sexpdata
    sfrbox-api
    sh
    sievelib
    simber
    simpleaudio
    simpleeval
    snapshottest
    soapysdr
    somajo
    speedtest-cli
    sphinx_pypi_upload
    sphinxcontrib-openapi
    sqlobject
    starkbank-ecdsa
    starlette
    staticjinja
    stdiomask
    strategies
    stravalib
    strenum
    strictyaml
    stringcase
    stringly
    sympy
    syncer
    sysv_ipc
    tabview
    takethetime
    tblite
    tcolorpy
    termstyle
    testing-common-database
    textacy
    textwrap3
    textx
    tweepy
    twilio
    twitter-common-collections
    twitter-common-confluence
    types-futures
    types-redis
    types-urllib3
    typesystem
    udatetime
    ukpostcodeparser
    unicrypto
    unidecode
    unidic-lite
    unpaddedbase64
    update-copyright
    ush
    vdirsyncer
    vector
    venusian
    versioneer
    veryprettytable
    videocr
    voluptuous-stubs
    volvooncall
    wakeonlan
    web
    webcolors
    webhelpers
    wfuzz
    whichcraft
    widlparser
    winacl
    wordfreq
    ws4py
    wsdiscovery
    wsgi-intercept
    xdg
    xhtml2pdf
    xstatic-jquery-file-upload
    xstatic-pygments
    yamllint
    yaramod
    yubico
    zarr
    zc_lockfile
    zigpy-zigate
    zigpy-znp
    zipstream
    zipstream-ng
    zope_proxy
    zope_schema
    zopfli
  ];
  /**/

  mkPdoc = use-pdoc3: drv: let
    isBuiltin = !lib.isDerivation drv;
    name      = if isBuiltin then drv       else drv.pname;
    desc      = if isBuiltin then "builtin" else drv.meta.description;
    version   = if isBuiltin then "-"       else drv.version;
    homepage  = if isBuiltin
      then "https://docs.python.org/3/library/${drv}.html"
      else drv.meta.homepage or "-";
    doc = pkgs.runCommand "pdoc${if use-pdoc3 then "3" else ""}-${name}-docs" {
      nativeBuildInputs = (if use-pdoc3
        then [pkgs.python3Packages.pdoc3]
        else [pkgs.python3Packages.pdoc])
      ++ lib.optionals (!isBuiltin) [ drv ]
      ++ lib.optionals (!isBuiltin) (lib.pipe (drv.passthru.optional-dependencies or {}) [
        builtins.attrValues
        lib.flatten
        (builtins.filter (drv':
          (builtins.tryEval drv'.outPath).success
        ))
      ]);

      env.NAME = lib.toLower name;
      env.DESC = lib.escapeXML desc;
      # TODO: license
      # TODO: build html with something better than bash
    } ''
      LITERALS=()
      ${lib.optionalString isBuiltin ''
        LITERALS+=("${name}")
        _tmp="$(python -c 'import ${name}; print((getattr(${name}, "__doc__", "") or "builtin").split("\n")[0])')"
        test "$?" -eq 0 && DESC="$_tmp"
      ''}
      ${lib.optionalString (!isBuiltin) ''
        LITERALS+=(${lib.escapeShellArgs (
            (drv.pythonImportsCheck or []) ++
            (drv.pythonImportsExtraCheck or [])
        )})

        pushd ${drv}/${pkgs.python3.sitePackages}
        shopt -s globstar
        #for fname in **/*; do
        for fname in *; do
          if test -f "$fname" && ( test "''${fname##*.}" = "py" || test "''${fname##*.}" = "so" ) ; then
            [[ "$fname" =~ (^|/)"_"* ]] && continue
            LITERALS+=("$(echo "''${fname%%.py*}" | tr "/-" "._" )")
          elif test -d "$fname" && test -f "$fname"/__init__.py; then
            LITERALS+=("$(echo "$fname" | tr "/-" "._" )")
          fi
        done
        popd

        # make unique
        LITERALS=( $(printf "%q\n" "''${LITERALS[@]}" | sort -u) )
        echo "''${LITERALS[0]}"
      ''}

      ( timeout 900s ${if !use-pdoc3
        then ''pdoc --no-search --math --no-browser --output-directory $out "''${LITERALS[@]}"''
        else ''pdoc3 --skip-errors --output-dir $out --html "''${LITERALS[@]}" --force''
      } 2>&1 | tee "$NAME".log ) || true
      mkdir -p $out
      cp "$NAME".log $out
      test -f $out/index.html && rm -v $out/index.html

      function write {
        { printf "%s" "$@"; echo; } >> $out/index.part-"$NAME".html
      }

      write "<tr>"
      if test -f $out/"''${LITERALS[0]}".html; then
        write "<td><a href=\"''${LITERALS[0]}.html\">${lib.escapeXML name}</a>"
      elif test -d $out/"''${LITERALS[0]}"; then
        write "<td><a href=\"''${LITERALS[0]}/\">${lib.escapeXML name}</a>"
      else
        write "<td>${lib.escapeXML name}"
      fi
      write "<td>${version}"
      if test -s $out/$NAME.log; then
        write "<td><a href=\"$NAME.log\">log</a>"
      else
        write "<td>-"
      fi
      write "<td>$DESC"
      ${if homepage == "-" then ''
        write "<td>n/a"
      '' else ''
        write "<td><a href=\"${homepage}\">${homepage}</a>"
      ''}
      write "</tr>"
    '';
    fallback = pkgs.writeTextDir "index.part-${lib.toLower name}.html" ''
      <tr>
      <td>${lib.escapeXML name}
      <td>${version}
      <td>&#10799;
      <td>${lib.escapeXML desc}
      <td>${if homepage == "-" then
        "n/a"
       else
        ''<a href="${homepage}">${homepage}</a>''
      }
      </tr>
    '';
  in if (builtins.tryEval doc.outPath).success
    then doc
    else fallback;

  mkPdocs = use-pdoc3: with builtins; pkgs.symlinkJoin {
    name = "pdoc-docs";
    paths = map (mkPdoc use-pdoc3) (python-builtins ++ python-packages);
    # note: globs are sorted
    postBuild = ''
      shopt -s nocaseglob
      >>$out/index.html echo "<!DOCTYPE html>"
      >>$out/index.html echo "<table><tr><th>name<th>version<th>log<th>description<th>homepage</tr>"
      >>$out/index.html cat $out/index.part-*.html
      >>$out/index.html echo "</table>"
      rm $out/index.part-*.html
    '';
  };
in {
  pdocs  = mkPdocs false;
  pdocs3 = mkPdocs true;
}