/** * WeChat article download — export WeChat Official Account articles to Markdown. * * Ported from jackwener/wechat-article-to-markdown (JS version) to OpenCLI adapter. * * Usage: * opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin */ import { cli, Strategy } from '@jackwener/opencli/registry'; import { downloadArticle } from '@jackwener/opencli/download/article-download'; // ============================================================ // URL Normalization // ============================================================ /** * Normalize a pasted WeChat article URL. */ // Wrapping quote characters to strip from a pasted URL. Covers ASCII plus // CJK typographic / smart quotes, which are common when users copy URLs from // Chinese-language environments (WeChat itself, macOS smart-quote // substitution, Word / Pages, …). // // Pairs: // " " ASCII straight double // ' ' ASCII straight single // “ ” curly double (U+201C / U+201D) // ‘ ’ curly single (U+2018 / U+2019) // 「 」 CJK corner brackets (U+300C / U+300D) // 『 』 CJK white corner brackets (U+300E / U+300F) // „ ‟ German-style double quotes (U+201E / U+201F) // ‹ › single guillemets (U+2039 / U+203A) // « » double guillemets (U+00AB / U+00BB) const WRAPPING_QUOTE_PAIRS = [ ['"', '"'], ["'", "'"], ['“', '”'], ['‘', '’'], ['「', '」'], ['『', '』'], ['„', '‟'], ['‹', '›'], ['«', '»'], ]; const LEADING_WRAP_CHARS = new Set(WRAPPING_QUOTE_PAIRS.map(([open]) => open).concat('<')); const TRAILING_WRAP_CHARS = new Set(WRAPPING_QUOTE_PAIRS.map(([, close]) => close).concat('>')); function stripBoundaryWrapChars(value) { let s = value; for (let i = 0; i < 4; i += 1) { const before = s; for (const [open, close] of WRAPPING_QUOTE_PAIRS) { if (s.length >= 2 && s.startsWith(open) && s.endsWith(close)) { s = s.slice(open.length, s.length - close.length).trim(); break; } } while (s && LEADING_WRAP_CHARS.has(s[0])) { s = s.slice(1).trimStart(); } while (s && TRAILING_WRAP_CHARS.has(s[s.length - 1])) { s = s.slice(0, -1).trimEnd(); } if (s === before) break; } return s; } export function normalizeWechatUrl(raw) { let s = (raw || '').trim(); if (!s) return s; // Strip quote / angle-bracket characters only at the pasted boundary. This // handles both paired wrappers ("", "“url”") and common one-sided // trailing punctuation ("url”") without touching encoded URL content. s = stripBoundaryWrapChars(s); // Remove backslash escapes before URL-significant characters s = s.replace(/\\+([:/&?=#%])/g, '$1'); // Decode HTML entities s = s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"'); // Allow bare hostnames if (s.startsWith('mp.weixin.qq.com/') || s.startsWith('//mp.weixin.qq.com/')) { s = 'https://' + s.replace(/^\/+/, ''); } // Force https for mp.weixin.qq.com try { const parsed = new URL(s); if (['http:', 'https:'].includes(parsed.protocol) && parsed.hostname.toLowerCase() === 'mp.weixin.qq.com') { parsed.protocol = 'https:'; s = parsed.toString(); } } catch { // Ignore parse errors } return s; } /** * Format a WeChat article timestamp as a UTC+8 datetime string. * Accepts either Unix seconds or milliseconds. */ export function formatWechatTimestamp(rawTimestamp) { const ts = Number.parseInt(rawTimestamp, 10); if (!Number.isFinite(ts) || ts <= 0) return ''; const timestampMs = rawTimestamp.length === 13 ? ts : ts * 1000; const d = new Date(timestampMs); const pad = (n) => String(n).padStart(2, '0'); const utc8 = new Date(d.getTime() + 8 * 3600 * 1000); return (`${utc8.getUTCFullYear()}-` + `${pad(utc8.getUTCMonth() + 1)}-` + `${pad(utc8.getUTCDate())} ` + `${pad(utc8.getUTCHours())}:` + `${pad(utc8.getUTCMinutes())}:` + `${pad(utc8.getUTCSeconds())}`); } /** * Extract the raw create_time value from supported WeChat inline script formats. */ export function extractWechatCreateTimeValue(htmlStr) { const jsDecodeMatch = htmlStr.match(/create_time\s*:\s*JsDecode\('([^']+)'\)(?=[\s,;}]|$)/); if (jsDecodeMatch) return jsDecodeMatch[1]; const directValueMatch = htmlStr.match(/create_time\s*[:=]\s*(?:"([^"]+)"|'([^']+)'|([0-9A-Za-z]+))(?=[\s,;}]|$)/); if (!directValueMatch) return ''; return directValueMatch[1] || directValueMatch[2] || directValueMatch[3] || ''; } /** * Extract the publish time from DOM text first, then fall back to numeric create_time values. */ export function extractWechatPublishTime(publishTimeText, htmlStr) { const normalizedPublishTime = (publishTimeText || '').trim(); if (normalizedPublishTime) return normalizedPublishTime; const rawCreateTime = extractWechatCreateTimeValue(htmlStr); if (!/^\d{10}$|^\d{13}$/.test(rawCreateTime)) return ''; return formatWechatTimestamp(rawCreateTime); } /** * Detect WeChat anti-bot / verification gate pages before we try to parse the article. */ export function detectWechatAccessIssue(pageText, htmlStr) { const normalizedText = (pageText || '').replace(/\s+/g, ' ').trim(); if (/环境异常/.test(normalizedText) && /(完成验证后即可继续访问|去验证)/.test(normalizedText)) { return 'environment verification required'; } if (/secitptpage\/verify\.html/.test(htmlStr) || /id=["']js_verify["']/.test(htmlStr)) { return 'environment verification required'; } return ''; } export function pickFirstWechatMetaText(...candidates) { for (const candidate of candidates) { const normalized = (candidate || '').replace(/\s+/g, ' ').trim(); if (normalized && normalized !== 'Name cleared') return normalized; } return ''; } /** * Build a self-contained helper for execution inside page.evaluate(). */ export function buildExtractWechatPublishTimeJs() { return `(${function extractWechatPublishTimeInPage(publishTimeText, htmlStr) { function formatWechatTimestamp(rawTimestamp) { const ts = Number.parseInt(rawTimestamp, 10); if (!Number.isFinite(ts) || ts <= 0) return ''; const timestampMs = rawTimestamp.length === 13 ? ts : ts * 1000; const d = new Date(timestampMs); const pad = (n) => String(n).padStart(2, '0'); const utc8 = new Date(d.getTime() + 8 * 3600 * 1000); return (`${utc8.getUTCFullYear()}-` + `${pad(utc8.getUTCMonth() + 1)}-` + `${pad(utc8.getUTCDate())} ` + `${pad(utc8.getUTCHours())}:` + `${pad(utc8.getUTCMinutes())}:` + `${pad(utc8.getUTCSeconds())}`); } function extractWechatCreateTimeValue(html) { const jsDecodeMatch = html.match(/create_time\s*:\s*JsDecode\('([^']+)'\)(?=[\s,;}]|$)/); if (jsDecodeMatch) return jsDecodeMatch[1]; const directValueMatch = html.match(/create_time\s*[:=]\s*(?:"([^"]+)"|'([^']+)'|([0-9A-Za-z]+))(?=[\s,;}]|$)/); if (!directValueMatch) return ''; return directValueMatch[1] || directValueMatch[2] || directValueMatch[3] || ''; } const normalizedPublishTime = (publishTimeText || '').trim(); if (normalizedPublishTime) return normalizedPublishTime; const rawCreateTime = extractWechatCreateTimeValue(htmlStr); if (!/^\d{10}$|^\d{13}$/.test(rawCreateTime)) return ''; return formatWechatTimestamp(rawCreateTime); }.toString()})`; } /** * Build a self-contained access-issue detector for execution inside page.evaluate(). */ export function buildDetectWechatAccessIssueJs() { return `(${function detectWechatAccessIssueInPage(pageText, htmlStr) { const normalizedText = (pageText || '').replace(/\s+/g, ' ').trim(); if (/环境异常/.test(normalizedText) && /(完成验证后即可继续访问|去验证)/.test(normalizedText)) { return 'environment verification required'; } if (/secitptpage\/verify\.html/.test(htmlStr) || /id=["']js_verify["']/.test(htmlStr)) { return 'environment verification required'; } return ''; }.toString()})`; } // ============================================================ // CLI Registration // ============================================================ cli({ site: 'weixin', name: 'download', access: 'read', description: '下载微信公众号文章为 Markdown 格式', domain: 'mp.weixin.qq.com', strategy: Strategy.COOKIE, args: [ { name: 'url', required: true, help: 'WeChat article URL (mp.weixin.qq.com/s/xxx)' }, { name: 'output', default: './weixin-articles', help: 'Output directory' }, { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' }, ], columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'], func: async (page, kwargs) => { const rawUrl = kwargs.url; const url = normalizeWechatUrl(rawUrl); if (!url.startsWith('https://mp.weixin.qq.com/')) { return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-', saved: '-' }]; } // Navigate and wait for content to load await page.goto(url); await page.wait(5); // Extract article data in browser context const data = await page.evaluate(` (() => { const result = { title: '', author: '', publishTime: '', errorHint: '', contentHtml: '', codeBlocks: [], imageUrls: [] }; const pickFirstText = (...selectors) => { for (const selector of selectors) { const text = document.querySelector(selector)?.textContent?.replace(/\\s+/g, ' ').trim() || ''; if (text && text !== 'Name cleared') return text; } return ''; }; // WeChat has multiple article templates. Newer pages use #js_text_title. result.title = pickFirstText( '#activity-name', '#js_text_title', '.rich_media_title', ); result.author = pickFirstText( '#js_name', '.wx_follow_nickname', '#profileBt .profile_nickname', '.rich_media_meta.rich_media_meta_nickname', '.rich_media_meta_nickname', ); // Publish time: prefer the rendered DOM text, then fall back to numeric create_time values. const publishTimeEl = document.querySelector('#publish_time'); const extractWechatPublishTime = ${buildExtractWechatPublishTimeJs()}; result.publishTime = extractWechatPublishTime( publishTimeEl ? publishTimeEl.textContent : '', document.documentElement.innerHTML, ); const detectWechatAccessIssue = ${buildDetectWechatAccessIssueJs()}; result.errorHint = detectWechatAccessIssue( document.body ? document.body.innerText : '', document.documentElement.innerHTML, ); if (result.errorHint) return result; // Content processing const contentEl = document.querySelector('#js_content'); if (!contentEl) return result; // Fix lazy-loaded images: data-src -> src contentEl.querySelectorAll('img').forEach(img => { const dataSrc = img.getAttribute('data-src'); if (dataSrc) img.setAttribute('src', dataSrc); }); // Extract code blocks with placeholder replacement const codeBlocks = []; contentEl.querySelectorAll('.code-snippet__fix').forEach(el => { el.querySelectorAll('.code-snippet__line-index').forEach(li => li.remove()); const pre = el.querySelector('pre[data-lang]'); const lang = pre ? (pre.getAttribute('data-lang') || '') : ''; const lines = []; el.querySelectorAll('code').forEach(codeTag => { const text = codeTag.textContent; if (/^[ce]?ounter\\(line/.test(text)) return; lines.push(text); }); if (lines.length === 0) lines.push(el.textContent); const placeholder = 'CODEBLOCK-PLACEHOLDER-' + codeBlocks.length; codeBlocks.push({ lang, code: lines.join('\\n') }); const p = document.createElement('p'); p.textContent = placeholder; el.replaceWith(p); }); result.codeBlocks = codeBlocks; // Remove noise elements ['script', 'style', '.qr_code_pc', '.reward_area'].forEach(sel => { contentEl.querySelectorAll(sel).forEach(tag => tag.remove()); }); // Collect image URLs (deduplicated) const seen = new Set(); contentEl.querySelectorAll('img[src]').forEach(img => { const src = img.getAttribute('src'); if (src && !seen.has(src)) { seen.add(src); result.imageUrls.push(src); } }); result.contentHtml = contentEl.innerHTML; return result; })() `); if (data?.errorHint === 'environment verification required') { return [{ title: 'Error', author: '-', publish_time: '-', status: 'failed — verification required in WeChat browser page', size: '-', saved: '-', }]; } return downloadArticle({ title: data?.title || '', author: data?.author, publishTime: data?.publishTime, sourceUrl: url, contentHtml: data?.contentHtml || '', codeBlocks: data?.codeBlocks, imageUrls: data?.imageUrls, }, { output: kwargs.output, downloadImages: kwargs['download-images'], imageHeaders: { Referer: 'https://mp.weixin.qq.com/' }, frontmatterLabels: { author: '公众号' }, detectImageExt: (url) => { const m = url.match(/wx_fmt=(\w+)/) || url.match(/\.(\w{3,4})(?:\?|$)/); return m ? m[1] : 'png'; }, }); }, });