Wiki Download

Download all Wiki Articles

In order to migrate to Markdown format, and most probable LeafWiki, I need to export all my Wiki pages.

Export/Download

wiki-download.py

import pathlib
import requests

API = "https://entorb.net/wiki/api.php"
OUT = pathlib.Path("export")
s = requests.Session() 

def all_titles(ns=0):
    params = {
        "action": "query",
        "list": "allpages",
        "aplimit": "max",
        "apnamespace": ns,
        "apfilterredir": "nonredirects",
        "format": "json",
    }
    while True:
        r = s.get(API, params=params).json()
        for p in r["query"]["allpages"]:
            yield p["title"]
        if "continue" not in r:
            break
        params.update(r["continue"])

def wikitext(title):
    r = s.get(
        API,
        params={
            "action": "query",
            "prop": "revisions",
            "rvprop": "content",
            "rvslots": "main",
            "titles": title,
            "format": "json",
            "formatversion": 2,
        },
    ).json()
    return r["query"]["pages"][0]["revisions"][0]["slots"]["main"]["content"]

def to_path(title):
    if title == "Python":
        return OUT / "Python/index"
    if title == "C++":
        return OUT / "Cpp"
    if title == "Notepad++":
        return OUT / "Notepad-plus-plus"
    name = title.replace("/", "-")  # a literal "/" never becomes a folder
    name = name.replace(" - ", "/")  # "Python - Math" -> Python/Math
    name = name.replace(" ", "-")
    return OUT / (name + ".wiki")

for title in all_titles(ns=0):
    path = to_path(title)
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(wikitext(title), encoding="utf-8")
    print(title, "->", path)

Convert to Markdown

requires pandoc

wiki2md.sh

#!/bin/bash

set -e

# no args: fan out one process per file; with arg: convert that file
if [ $# -eq 0 ]; then
  find export -name '*.wiki' -print0 | xargs -0 -n1 -P 4 "$0"
  exit
fi
f=$1
echo "$f"
out="md/${f#export/}"
out="${out%.wiki}.md"
mkdir -p "$(dirname "$out")"
title="${f#export/}"
title="${title%.wiki}"
{
  title="${title//_/ }"
  echo "# ${title//\// - }"
  echo
  tags=$(grep -o '\[\[Category:[^]|]*' "$f" | sed 's/.*Category://; s/ /_/g; s/^/#/' | tr '\n' ' ' || true)
  [ -n "$tags" ] && printf 'Tags: %s\n\n' "${tags% }"
  pandoc -f mediawiki -t gfm --wrap=none --lua-filter=wiki2md.lua "$f" |
    # opening fence: "``` lang" -> "```lang", bare "```" -> "```plain" (pandoc drops "plain")
    awk '/^```/ { if (!f) { sub(/ /, ""); if ($0 ~ /^`+$/) $0 = $0 "plain" } f = !f } 1'
} >"$out"

wiki2md.lua

-- pandoc renders mediawiki " "-indented blocks as inline code + line breaks; turn them into code blocks

-- untagged code blocks (e.g. from

) get "plain", so gfm writes them fenced
 function CodeBlock(el)
   if #el.classes == 0 then
     el.classes = { "plain" }
     return el
   end
 end
 
 -- categories are emitted as #tags by wiki2md.sh
 function Link(el)
   if el.target:match("^Category:") then
     return {}
   end
 end
 
 -- text of a Code, or of an autolinked URL inside code (Link of Code only); nil otherwise
 local function code_text(x)
   if x.t == "Code" then
     return x.text
   end
   if x.t == "Link" then
     local s = ""
     for _, y in ipairs(x.content) do
       if y.t ~= "Code" then
         return nil
       end
       s = s .. y.text
     end
     return s
   end
 end
 
 function Para(el)
   if #el.content == 0 then
     return {}
   end
   local lines, cur, has_code = {}, "", false
   for _, x in ipairs(el.content) do
     local text = code_text(x)
     if text then
       cur = cur .. text:gsub("\u{A0}", " ")
       has_code = true
     elseif x.t == "Space" then
       cur = cur .. " "
     elseif x.t == "Str" and x.text:match("^%s*$") then
       cur = cur .. x.text
     elseif x.t == "LineBreak" then
       table.insert(lines, cur)
       cur = ""
     else
       return nil
     end
   end
   if not has_code then
     return nil
   end
   table.insert(lines, cur)
   while #lines > 0 and lines[#lines]:match("^%s*$") do
     table.remove(lines)
   end
   return pandoc.CodeBlock(table.concat(lines, "\n"), pandoc.Attr("", { "plain" }))
 end