Add ESLint configuration, .gitignore, Deno configuration, and initial Jupyter notebook
This commit is contained in:
+312
@@ -0,0 +1,312 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "9f587bf1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import { load } from \"jsr:@std/dotenv\";\n",
|
||||
"import OpenAI from \"jsr:@openai/openai\";\n",
|
||||
"\n",
|
||||
"const _ = await load({ export: true });\n",
|
||||
"const openai = new OpenAI();"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "4650126c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"const safeName = (s: string) => s.replace(/[<>:\"/\\\\|?*\\x00-\\x1F]/g, \"_\").trim();\n",
|
||||
"const bookName =\n",
|
||||
" \"Nmap Network Scanning Official Nmap Project Guide to Network Discovery and Security Scanning\";\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "ae701b32",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"const ordinals = [\n",
|
||||
" \"first\",\n",
|
||||
" \"second\",\n",
|
||||
" \"third\",\n",
|
||||
" \"fourth\",\n",
|
||||
" \"fifth\",\n",
|
||||
" \"sixth\",\n",
|
||||
" \"seventh\",\n",
|
||||
" \"eighth\",\n",
|
||||
" \"ninth\",\n",
|
||||
" \"tenth\",\n",
|
||||
" \"eleventh\",\n",
|
||||
" \"twelfth\",\n",
|
||||
" \"thirteenth\",\n",
|
||||
" \"fourteenth\",\n",
|
||||
" \"fifteenth\",\n",
|
||||
" \"sixteenth\",\n",
|
||||
" \"seventeenth\",\n",
|
||||
" \"eighteenth\",\n",
|
||||
" \"nineteenth\",\n",
|
||||
" \"twentieth\",\n",
|
||||
"];\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8bee369d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[\n",
|
||||
" { title: \"01\", level: 1, page: 27, endpage: 50 },\n",
|
||||
" { title: \"02\", level: 1, page: 51, endpage: 71 },\n",
|
||||
" { title: \"03\", level: 1, page: 72, endpage: 97 },\n",
|
||||
" { title: \"04\", level: 1, page: 98, endpage: 119 },\n",
|
||||
" { title: \"05\", level: 1, page: 120, endpage: 158 },\n",
|
||||
" { title: \"06\", level: 1, page: 159, endpage: 168 },\n",
|
||||
" { title: \"07\", level: 1, page: 169, endpage: 193 },\n",
|
||||
" { title: \"08\", level: 1, page: 194, endpage: 227 },\n",
|
||||
" { title: \"09\", level: 1, page: 228, endpage: 278 },\n",
|
||||
" { title: \"10\", level: 1, page: 279, endpage: 315 },\n",
|
||||
" { title: \"11\", level: 1, page: 316, endpage: 326 },\n",
|
||||
" { title: \"12\", level: 1, page: 327, endpage: 356 },\n",
|
||||
" { title: \"13\", level: 1, page: 357, endpage: 382 },\n",
|
||||
" { title: \"14\", level: 1, page: 383, endpage: 391 },\n",
|
||||
" { title: \"15\", level: 1, page: 392, endpage: 392 }\n",
|
||||
"]\n",
|
||||
"Processing chapter: 01\n",
|
||||
"Processing chapter: 02\n",
|
||||
"Processing chapter: 03\n",
|
||||
"Processing chapter: 04\n",
|
||||
"Processing chapter: 05\n",
|
||||
"Processing chapter: 06\n",
|
||||
"Processing chapter: 07\n",
|
||||
"Processing chapter: 08\n",
|
||||
"Processing chapter: 09\n",
|
||||
"Processing chapter: 10\n",
|
||||
"Processing chapter: 11\n",
|
||||
"Processing chapter: 12\n",
|
||||
"Processing chapter: 13\n",
|
||||
"Processing chapter: 14\n",
|
||||
"Processing chapter: 15\n",
|
||||
"Finished processing chapter: 15\n",
|
||||
"Finished processing chapter: 11\n",
|
||||
"Finished processing chapter: 14\n",
|
||||
"Finished processing chapter: 05\n",
|
||||
"Finished processing chapter: 07\n",
|
||||
"Finished processing chapter: 06\n",
|
||||
"Finished processing chapter: 12\n",
|
||||
"Finished processing chapter: 04\n",
|
||||
"Finished processing chapter: 03\n",
|
||||
"Finished processing chapter: 01\n",
|
||||
"Finished processing chapter: 13\n",
|
||||
"Finished processing chapter: 09\n",
|
||||
"Finished processing chapter: 08\n",
|
||||
"Finished processing chapter: 02\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import { PDFDocument } from \"https://cdn.skypack.dev/pdf-lib@^1.11.1?dts\";\n",
|
||||
"\n",
|
||||
"const INPUT_PDF_PATH = `./books/${bookName}.pdf`;\n",
|
||||
"\n",
|
||||
"type Bookmark = { title: string; page: number; level: number; endpage: number };\n",
|
||||
"const bookmarks = await new Deno.Command(\"pdftk\", {\n",
|
||||
" args: [\n",
|
||||
" INPUT_PDF_PATH,\n",
|
||||
" \"dump_data\",\n",
|
||||
" \"output\",\n",
|
||||
" \"-\",\n",
|
||||
" ],\n",
|
||||
"}).output().then(\n",
|
||||
" (res) => new TextDecoder().decode(res.stdout),\n",
|
||||
").then((data) => {\n",
|
||||
" const lines = data.split(\"\\n\");\n",
|
||||
" let bookmarks = [];\n",
|
||||
" let currentBookmark: Partial<Bookmark> | null = null;\n",
|
||||
"\n",
|
||||
" for (const line of lines) {\n",
|
||||
" if (line.startsWith(\"BookmarkBegin\")) {\n",
|
||||
" if (currentBookmark) {\n",
|
||||
" bookmarks.push(currentBookmark as Bookmark);\n",
|
||||
" }\n",
|
||||
" currentBookmark = {};\n",
|
||||
" } else if (line.startsWith(\"BookmarkTitle:\")) {\n",
|
||||
" if (currentBookmark) {\n",
|
||||
" currentBookmark.title = line.replace(\"BookmarkTitle: \", \"\").trim();\n",
|
||||
" }\n",
|
||||
" } else if (line.startsWith(\"BookmarkLevel:\")) {\n",
|
||||
" if (currentBookmark) {\n",
|
||||
" currentBookmark.level = parseInt(\n",
|
||||
" line.replace(\"BookmarkLevel: \", \"\").trim(),\n",
|
||||
" );\n",
|
||||
" }\n",
|
||||
" } else if (line.startsWith(\"BookmarkPageNumber:\")) {\n",
|
||||
" if (currentBookmark) {\n",
|
||||
" currentBookmark.page = parseInt(\n",
|
||||
" line.replace(\"BookmarkPageNumber: \", \"\").trim(),\n",
|
||||
" );\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
" if (currentBookmark) {\n",
|
||||
" bookmarks.push(currentBookmark as Bookmark);\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" bookmarks = bookmarks.filter((b) => b.level === 1).slice(8);\n",
|
||||
"\n",
|
||||
" for (let i = 0; i < bookmarks.length; i++) {\n",
|
||||
" const current = bookmarks[i] as Bookmark;\n",
|
||||
" const next = bookmarks[i + 1] as Bookmark | undefined;\n",
|
||||
" const currentPage = current.page ?? 0;\n",
|
||||
" const nextPage = next?.page ?? 0;\n",
|
||||
" current.endpage = nextPage\n",
|
||||
" ? Math.max(currentPage, nextPage - 1)\n",
|
||||
" : currentPage;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" return bookmarks;\n",
|
||||
"});\n",
|
||||
"console.log(bookmarks);\n",
|
||||
"\n",
|
||||
"const promises = [];\n",
|
||||
"for (const [idx, ch] of bookmarks.entries()) {\n",
|
||||
" async function processChapter(idx: number, title: string, file: File) {\n",
|
||||
" const upload = await openai.files.create({\n",
|
||||
" file,\n",
|
||||
" purpose: \"user_data\",\n",
|
||||
" });\n",
|
||||
"\n",
|
||||
" const response = await openai.responses.create({\n",
|
||||
" model: \"gpt-5.2\",\n",
|
||||
" reasoning: { effort: \"xhigh\" },\n",
|
||||
" input: [{\n",
|
||||
" role: \"user\",\n",
|
||||
" content: [\n",
|
||||
" {\n",
|
||||
" type: \"input_text\",\n",
|
||||
" text:\n",
|
||||
" `A complete, exhaustive and very detailed mind map in markmap syntax of only and only this ${\n",
|
||||
" ordinals[idx]\n",
|
||||
" } chapter`,\n",
|
||||
" },\n",
|
||||
" { type: \"input_file\", file_id: upload.id },\n",
|
||||
" ],\n",
|
||||
" }],\n",
|
||||
" metadata: {\n",
|
||||
" chapter: title,\n",
|
||||
" ordinal: ordinals[idx],\n",
|
||||
" },\n",
|
||||
" });\n",
|
||||
" console.log(`Processing chapter: ${title} at ${response.id}`);\n",
|
||||
"\n",
|
||||
" await openai.files.delete(upload.id);\n",
|
||||
"\n",
|
||||
" const mindMapContent = response.output_text;\n",
|
||||
" await Deno.writeTextFile(\n",
|
||||
" `./mindmaps/${safeName(title)}.md`,\n",
|
||||
" mindMapContent,\n",
|
||||
" );\n",
|
||||
"\n",
|
||||
" console.log(`Finished processing chapter: ${title}`);\n",
|
||||
" }\n",
|
||||
" {\n",
|
||||
" // Split the chapter into a separate PDF file\n",
|
||||
" const pdfBytes = await Deno.readFile(INPUT_PDF_PATH);\n",
|
||||
" const pdfDoc = await PDFDocument.load(pdfBytes);\n",
|
||||
" const chapterPdf = await PDFDocument.create();\n",
|
||||
"\n",
|
||||
" const startPage = ch.page - 1; // zero-based index\n",
|
||||
" const endPage = ch.endpage - 1; // zero-based index\n",
|
||||
"\n",
|
||||
" const pagesToCopy = await chapterPdf.copyPages(\n",
|
||||
" pdfDoc,\n",
|
||||
" Array.from(\n",
|
||||
" { length: endPage - startPage + 1 },\n",
|
||||
" (_, i) => i + startPage,\n",
|
||||
" ),\n",
|
||||
" );\n",
|
||||
"\n",
|
||||
" type PdfPage =\n",
|
||||
" import(\"https://cdn.skypack.dev/pdf-lib@^1.11.1?dts\").PDFPage;\n",
|
||||
"\n",
|
||||
" pagesToCopy.forEach((page: PdfPage): void => {\n",
|
||||
" chapterPdf.addPage(page);\n",
|
||||
" });\n",
|
||||
"\n",
|
||||
" const chapterPdfBytes = await chapterPdf.save();\n",
|
||||
" const chapterFile = new File(\n",
|
||||
" [chapterPdfBytes],\n",
|
||||
" `${safeName(ch.title)}.pdf`,\n",
|
||||
" { type: \"application/pdf\" },\n",
|
||||
" );\n",
|
||||
" promises.push(processChapter(idx, ch.title, chapterFile));\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"await Promise.all(promises);\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "7333f395",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{ object: \u001b[32m\"file\"\u001b[39m, deleted: \u001b[33mtrue\u001b[39m, id: \u001b[32m\"file-X2CJi3gozhJniBURG2qfsm\"\u001b[39m }"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"// delete all files from openai file storage\n",
|
||||
"const userFiles = await openai.files.list({ purpose: \"user_data\" });\n",
|
||||
"for (const file of userFiles.data) {\n",
|
||||
" await openai.files.delete(file.id);\n",
|
||||
"}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b8a5dc2c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Deno",
|
||||
"language": "typescript",
|
||||
"name": "deno"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": "typescript",
|
||||
"file_extension": ".ts",
|
||||
"mimetype": "text/x.typescript",
|
||||
"name": "typescript",
|
||||
"nbconvert_exporter": "script",
|
||||
"pygments_lexer": "typescript",
|
||||
"version": "5.9.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in New Issue
Block a user