Wiki Download - Revision history

Torben: /* Convert to Markdown */

2026-10-25T19:22:54Z

Convert to Markdown

← Older revision		Revision as of 20:22, 25 October 2026
Line 95:		Line 95:
	wiki2md.lua		wiki2md.lua
	-- pandoc renders mediawiki " "-indented blocks as inline code + line breaks; turn them into code blocks		-- pandoc renders mediawiki " "-indented blocks as inline code + line breaks; turn them into code blocks

	-- untagged code blocks (e.g. from < pre >) get "plain", so gfm writes them fenced		-- untagged code blocks (e.g. from < pre >) get "plain", so gfm writes them fenced

Torben: /* Convert to Markdown */

2026-10-25T19:22:43Z

Convert to Markdown

← Older revision		Revision as of 20:22, 25 October 2026
Line 96:		Line 96:
	-- pandoc renders mediawiki " "-indented blocks as inline code + line breaks; turn them into code blocks		-- pandoc renders mediawiki " "-indented blocks as inline code + line breaks; turn them into code blocks

	-- untagged code blocks (e.g. from <pre>) get "plain", so gfm writes them fenced		-- untagged code blocks (e.g. from < pre >) get "plain", so gfm writes them fenced

	function CodeBlock(el)		function CodeBlock(el)

Torben: /* Convert to Markdown */

2026-10-25T19:22:24Z

Convert to Markdown

← Older revision		Revision as of 20:22, 25 October 2026
Line 95:		Line 95:
	wiki2md.lua		wiki2md.lua
	-- pandoc renders mediawiki " "-indented blocks as inline code + line breaks; turn them into code blocks		-- pandoc renders mediawiki " "-indented blocks as inline code + line breaks; turn them into code blocks

	-- untagged code blocks (e.g. from <pre>) get "plain", so gfm writes them fenced		-- untagged code blocks (e.g. from <pre>) get "plain", so gfm writes them fenced

	function CodeBlock(el)		function CodeBlock(el)
	if #el.classes == 0 then		if #el.classes == 0 then

Torben: /* Download all Wiki Articles */

2026-10-25T19:21:17Z

Download all Wiki Articles

← Older revision		Revision as of 20:21, 25 October 2026
Line 1:		Line 1:
	== Download all Wiki Articles ==		== Download all Wiki Articles ==
	In order to migrate to Markdown format, and most probable LeafWiki, I need to export all my Wiki pages.		In order to migrate to Markdown format, and most probable [https://github.com/perber/leafwiki LeafWiki], I need to export all my Wiki pages.

	===Export/Download===		===Export/Download===

Torben: /* Convert to Markdown */

2026-10-25T19:20:37Z

Convert to Markdown

← Older revision		Revision as of 20:20, 25 October 2026
Line 61:		Line 61:
	print(title, "->", path)		print(title, "->", path)

	==Convert to Markdown==		===Convert to Markdown===
	requires pandoc		requires pandoc

Torben: Created page with "== Download all Wiki Articles == In order to migrate to Markdown format, and most probable LeafWiki, I need to export all my Wiki pages. ===Export/Download=== wiki-download.py import pathlib import requests API = "https://entorb.net/wiki/api.php" OUT = pathlib.Path("export") s = requests.Session() def all_titles(ns=0): params = { "action": "query", "list": "allpages", "aplimit": "max", "apnamespace": ns, "apfi..."

2026-10-25T19:20:24Z

Created page with "== Download all Wiki Articles == In order to migrate to Markdown format, and most probable LeafWiki, I need to export all my Wiki pages. ===Export/Download=== wiki-download.py import pathlib import requests API = "https://entorb.net/wiki/api.php" OUT = pathlib.Path("export") s = requests.Session() def all_titles(ns=0): params = { "action": "query", "list": "allpages", "aplimit": "max", "apnamespace": ns, "apfi..."

New page

== Download all Wiki Articles ==
In order to migrate to Markdown format, and most probable LeafWiki, I need to export all my Wiki pages.

===Export/Download===
wiki-download.py
import pathlib
import requests

API = "https://entorb.net/wiki/api.php"
OUT = pathlib.Path("export")
s = requests.Session()

def all_titles(ns=0):
params = {
"action": "query",
"list": "allpages",
"aplimit": "max",
"apnamespace": ns,
"apfilterredir": "nonredirects",
"format": "json",
}
while True:
r = s.get(API, params=params).json()
for p in r["query"]["allpages"]:
yield p["title"]
if "continue" not in r:
break
params.update(r["continue"])

def wikitext(title):
r = s.get(
API,
params={
"action": "query",
"prop": "revisions",
"rvprop": "content",
"rvslots": "main",
"titles": title,
"format": "json",
"formatversion": 2,
},
).json()
return r["query"]["pages"][0]["revisions"][0]["slots"]["main"]["content"]

def to_path(title):
if title == "Python":
return OUT / "Python/index"
if title == "C++":
return OUT / "Cpp"
if title == "Notepad++":
return OUT / "Notepad-plus-plus"
name = title.replace("/", "-") # a literal "/" never becomes a folder
name = name.replace(" - ", "/") # "Python - Math" -> Python/Math
name = name.replace(" ", "-")
return OUT / (name + ".wiki")

for title in all_titles(ns=0):
path = to_path(title)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(wikitext(title), encoding="utf-8")
print(title, "->", path)

==Convert to Markdown==
requires pandoc

wiki2md.sh
#!/bin/bash

set -e

# no args: fan out one process per file; with arg: convert that file
if [ $# -eq 0 ]; then
find export -name '*.wiki' -print0 | xargs -0 -n1 -P 4 "$0"
exit
fi
f=$1
echo "$f"
out="md/${f#export/}"
out="${out%.wiki}.md"
mkdir -p "$(dirname "$out")"
title="${f#export/}"
title="${title%.wiki}"
{
title="${title//_/ }"
echo "# ${title//\// - }"
echo
tags=$(grep -o '\[\[Category:[^]|]*' "$f" | sed 's/.*Category://; s/ /_/g; s/^/#/' | tr '\n' ' ' || true)
[ -n "$tags" ] && printf 'Tags: %s\n\n' "${tags% }"
pandoc -f mediawiki -t gfm --wrap=none --lua-filter=wiki2md.lua "$f" |
# opening fence: "``` lang" -> "```lang", bare "```" -> "```plain" (pandoc drops "plain")
awk '/^```/ { if (!f) { sub(/ /, ""); if ($0 ~ /^`+$/) $0 = $0 "plain" } f = !f } 1'
} >"$out"

wiki2md.lua
-- pandoc renders mediawiki " "-indented blocks as inline code + line breaks; turn them into code blocks
-- untagged code blocks (e.g. from <pre>) get "plain", so gfm writes them fenced
function CodeBlock(el)
if #el.classes == 0 then
el.classes = { "plain" }
return el
end
end

-- categories are emitted as #tags by wiki2md.sh
function Link(el)
if el.target:match("^Category:") then
return {}
end
end

-- text of a Code, or of an autolinked URL inside code (Link of Code only); nil otherwise
local function code_text(x)
if x.t == "Code" then
return x.text
end
if x.t == "Link" then
local s = ""
for _, y in ipairs(x.content) do
if y.t ~= "Code" then
return nil
end
s = s .. y.text
end
return s
end
end

function Para(el)
if #el.content == 0 then
return {}
end
local lines, cur, has_code = {}, "", false
for _, x in ipairs(el.content) do
local text = code_text(x)
if text then
cur = cur .. text:gsub("\u{A0}", " ")
has_code = true
elseif x.t == "Space" then
cur = cur .. " "
elseif x.t == "Str" and x.text:match("^%s*$") then
cur = cur .. x.text
elseif x.t == "LineBreak" then
table.insert(lines, cur)
cur = ""
else
return nil
end
end
if not has_code then
return nil
end
table.insert(lines, cur)
while #lines > 0 and lines[#lines]:match("^%s*$") do
table.remove(lines)
end
return pandoc.CodeBlock(table.concat(lines, "\n"), pandoc.Attr("", { "plain" }))
end