Wiki Download

Download all Wiki Articles

In order to migrate to Markdown format, and most probable LeafWiki, I need to export all my Wiki pages.

Export/Download

wiki-download.py

import pathlib
import requests

API = "https://entorb.net/wiki/api.php"
OUT = pathlib.Path("export")
s = requests.Session() 

def all_titles(ns=0):
    params = {
        "action": "query",
        "list": "allpages",
        "aplimit": "max",
        "apnamespace": ns,
        "apfilterredir": "nonredirects",
        "format": "json",
    }
    while True:
        r = s.get(API, params=params).json()
        for p in r["query"]["allpages"]:
            yield p["title"]
        if "continue" not in r:
            break
        params.update(r["continue"])

def wikitext(title):
    r = s.get(
        API,
        params={
            "action": "query",
            "prop": "revisions",
            "rvprop": "content",
            "rvslots": "main",
            "titles": title,
            "format": "json",
            "formatversion": 2,
        },
    ).json()
    return r["query"]["pages"][0]["revisions"][0]["slots"]["main"]["content"]

def to_path(title):
    if title == "Python":
        return OUT / "Python/index"
    if title == "C++":
        return OUT / "Cpp"
    if title == "Notepad++":
        return OUT / "Notepad-plus-plus"
    name = title.replace("/", "-")  # a literal "/" never becomes a folder
    name = name.replace(" - ", "/")  # "Python - Math" -> Python/Math
    name = name.replace(" ", "-")
    return OUT / (name + ".wiki")

for title in all_titles(ns=0):
    path = to_path(title)
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(wikitext(title), encoding="utf-8")
    print(title, "->", path)

Convert to Markdown

requires pandoc

wiki2md.sh

#!/bin/bash

set -e

# no args: fan out one process per file; with arg: convert that file
if [ $# -eq 0 ]; then
  find export -name '*.wiki' -print0 | xargs -0 -n1 -P 4 "$0"
  exit
fi
f=$1
echo "$f"
out="md/${f#export/}"
out="${out%.wiki}.md"
mkdir -p "$(dirname "$out")"
title="${f#export/}"
title="${title%.wiki}"
{
  title="${title//_/ }"
  echo "# ${title//\// - }"
  echo
  tags=$(grep -o '\[\[Category:[^]|]*' "$f" | sed 's/.*Category://; s/ /_/g; s/^/#/' | tr '\n' ' ' || true)
  [ -n "$tags" ] && printf 'Tags: %s\n\n' "${tags% }"
  pandoc -f mediawiki -t gfm --wrap=none --lua-filter=wiki2md.lua "$f" |
    # opening fence: "``` lang" -> "```lang", bare "```" -> "```plain" (pandoc drops "plain")
    awk '/^```/ { if (!f) { sub(/ /, ""); if ($0 ~ /^`+$/) $0 = $0 "plain" } f = !f } 1'
} >"$out"

wiki2md.lua

-- pandoc renders mediawiki " "-indented blocks as inline code + line breaks; turn them into code blocks
-- untagged code blocks (e.g. from < pre >) get "plain", so gfm writes them fenced

function CodeBlock(el)
  if #el.classes == 0 then
    el.classes = { "plain" }
    return el
  end
end

-- categories are emitted as #tags by wiki2md.sh
function Link(el)
  if el.target:match("^Category:") then
    return {}
  end
end

-- text of a Code, or of an autolinked URL inside code (Link of Code only); nil otherwise
local function code_text(x)
  if x.t == "Code" then
    return x.text
  end
  if x.t == "Link" then
    local s = ""
    for _, y in ipairs(x.content) do
      if y.t ~= "Code" then
        return nil
      end
      s = s .. y.text
    end
    return s
  end
end

function Para(el)
  if #el.content == 0 then
    return {}
  end
  local lines, cur, has_code = {}, "", false
  for _, x in ipairs(el.content) do
    local text = code_text(x)
    if text then
      cur = cur .. text:gsub("\u{A0}", " ")
      has_code = true
    elseif x.t == "Space" then
      cur = cur .. " "
    elseif x.t == "Str" and x.text:match("^%s*$") then
      cur = cur .. x.text
    elseif x.t == "LineBreak" then
      table.insert(lines, cur)
      cur = ""
    else
      return nil
    end
  end
  if not has_code then
    return nil
  end
  table.insert(lines, cur)
  while #lines > 0 and lines[#lines]:match("^%s*$") do
    table.remove(lines)
  end
  return pandoc.CodeBlock(table.concat(lines, "\n"), pandoc.Attr("", { "plain" }))
end