Skip to content

Commit f1a2cb9

Browse files
hoothinhoothin
committed
Update DownloadAllContent.user.js
1 parent 2325ff0 commit f1a2cb9

1 file changed

Lines changed: 224 additions & 52 deletions

File tree

‎DownloadAllContent/DownloadAllContent.user.js‎

Lines changed: 224 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
// @name:zh-TW 懶人小説下載器
55
// @name:ja 怠惰者小説ダウンロードツール
66
// @namespace hoothin
7-
// @version 1.28
7+
// @version 2.0
88
// @description Fetch and download main content on current page, provide special support for chinese novel
99
// @description:zh-CN 通用网站内容抓取工具，可批量抓取小说、论坛内容等并保存为TXT文档
1010
// @description:zh-TW 通用網站內容抓取工具，可批量抓取小說、論壇內容等並保存為TXT文檔
@@ -27,22 +27,45 @@
2727

2828
(function() {
2929
'use strict';
30+
var downThreadNum = 20;//下載綫程數
31+
var indexReg=/PART\b|Prologue|-\d+|分卷|Chapter\s*[\-_]?\d+|^序$|序\s*言|序\s*章|前\s*言|引\s*言|引\s*子|摘\s*要|楔\s*子|契\s*子|后\s*记|後\s*記|附\s*言|结\s*语|結\s*語|最終話|最终话|番\s*外|[\d|〇|零|一|二|三|四|五|六|七|八|九|十|百|千|万|萬|-]+\s*(、|）|\.\D|章|节|節|回|卷|折|篇|幕|集|话|話)/i;
32+
var innerNextPage=/下一(页|张)|next page/i;
3033
var lang = navigator.appName=="Netscape"?navigator.language:navigator.userLanguage;
3134
var i18n={};
3235
var rCats=[];
3336
switch (lang){
3437
case "zh-CN":
38+
case "zh-SG":
3539
i18n={
3640
fetch:"开始下载小说或其他【Ctrl+F9】",
37-
info:"本文是使用懒人小说下载器（DownloadAllContent）脚本下载的",
41+
info:"本文是使用懒人小说下载器（DownloadAllContent）下载的",
3842
error:"该段内容获取失败",
3943
downloading:"已下载完成 %s 段，剩余 %s 段<br>正在下载 %s",
4044
complete:"已全部下载完成，共 %s 段",
41-
del:"设置小说干扰码的CSS选择器",
42-
custom:"自定义网址下载",
45+
del:"设置文本干扰码的CSS选择器",
46+
custom:"自定义下载",
47+
customInfo:"输入网址或者章节CSS选择器",
4348
reSort:"按标题名重新排序",
44-
reSortUrl:"按网址重新排序（优先）",
45-
setting:"懒人小说下载设置"
49+
setting:"懒人小说下载设置",
50+
abort:"跳过此章",
51+
save:"临时保存"
52+
};
53+
break;
54+
case "zh-TW":
55+
case "zh-HK":
56+
i18n={
57+
fetch:"開始下載小說或其他【Ctrl+F9】",
58+
info:"本文是使用懶人小說下載器（DownloadAllContent）下載的",
59+
error:"該段內容獲取失敗",
60+
downloading:"已下載完成 %s 段，剩餘 %s 段<br>正在下載 %s",
61+
complete:"已全部下載完成，共 %s 段",
62+
del:"設置文本干擾碼的CSS選擇器",
63+
custom:"自定義下載",
64+
customInfo:"輸入網址或者章節CSS選擇器",
65+
reSort:"按標題名重新排序",
66+
setting:"懶人小說下載設置",
67+
abort:"跳過此章",
68+
save:"保存當前"
4669
};
4770
break;
4871
default:
@@ -51,16 +74,18 @@
5174
info:"The TXT is downloaded by 'DownloadAllContent'",
5275
error:"Failed in downloading current chapter",
5376
downloading:"%s pages are downloaded, there are still %s pages left<br>Downloading %s ......",
54-
complete:"Completed! The pages totalled %s",
55-
del:"Set css selectors for delete",
56-
custom:"Custom url for download",
57-
reSort:"Resort by titles",
58-
reSortUrl:"Resort by URLs (higher priority)",
59-
setting:"DownloadAllContent Setting"
77+
complete:"Completed! Get %s pages in total",
78+
del:"Set css selectors for ignore",
79+
custom:"Custom to download",
80+
customInfo:"Input urls OR sss selectors for chapter links",
81+
reSort:"ReSort by title",
82+
setting:"DownloadAllContent Setting",
83+
abort:"Abort",
84+
save:"Save"
6085
};
6186
break;
6287
}
63-
var firefox=navigator.userAgent.toLowerCase().indexOf('firefox')!=-1;
88+
var firefox=navigator.userAgent.toLowerCase().indexOf('firefox')!=-1,curRequests=[];
6489
var rocketContent,txtDownContent,txtDownWords,txtDownQuit,txtDownDivInited=false;
6590

6691
function initTxtDownDiv(){
@@ -70,14 +95,15 @@
7095
document.body.appendChild(rocketContent);
7196
rocketContent.outerHTML=`
7297
<div id="txtDownContent" style="display: none;">
73-
<div style="width:360px;height:70px;position:fixed;left:50%;top:50%;margin-top:-25px;margin-left:-150px;z-index:100000;background-color:#ffffff;border:1px solid #afb3b6;border-radius:10px;opacity:0.95;filter:alpha(opacity=95);box-shadow:5px 5px 20px 0px #000;">
74-
<div id="txtDownWords" style="position:absolute;width:280px;height: 50px;border: 1px solid #f3f1f1;padding: 8px;border-radius: 10px;">
98+
<div style="width:360px;height:90px;position:fixed;left:50%;top:50%;margin-top:-25px;margin-left:-150px;z-index:100000;background-color:#ffffff;border:1px solid #afb3b6;border-radius:10px;opacity:0.95;filter:alpha(opacity=95);box-shadow:5px 5px 20px 0px #000;">
99+
<div id="txtDownWords" style="position:absolute;width:275px;height: 100%;border: 1px solid #f3f1f1;padding: 8px;border-radius: 10px;overflow: auto;">
75100
</div>
76101
<div id="txtDownQuit" style="width:36px;height:28px;border-radius:10px;position:absolute;right:2px;top:2px;cursor: pointer;background-color:#ff5a5a;">
77102
<span style="height:28px;line-height:28px;display:block;color:#FFF;text-align:center;font-size:20px;">╳</span>
78103
</div>
79-
<div style="position:absolute;right:2px;bottom:2px;cursor: pointer;">
80-
<button id="tempSaveTxt" style="background: #008aff;border: 0;padding: 5px;border-radius: 10px;color: white;">保存</button>
104+
<div style="position:absolute;right:0px;bottom:2px;cursor: pointer;width:68px">
105+
<button id="abortRequest" style="background: #008aff;border: 0;padding: 5px;border-radius: 10px;color: white;float: right;margin: 1px;">${getI18n('abort')}</button>
106+
<button id="tempSaveTxt" style="background: #008aff;border: 0;padding: 5px;border-radius: 10px;color: white;float: right;margin: 1px;">${getI18n('save')}</button>
81107
</div>
82108
</div>
83109
</div>`;
@@ -93,17 +119,96 @@
93119

94120
function initTempSave(){
95121
var tempSavebtn = document.getElementById('tempSaveTxt');
122+
var abortbtn = document.getElementById('abortRequest');
96123
tempSavebtn.onclick = function(){
97124
var blob = new Blob([i18n.info+"\r\n"+document.title+"\r\n\r\n"+rCats.join("\r\n\r\n")], {type: "text/plain;charset=utf-8"});
98125
saveAs(blob, document.title+".txt");
99126
}
127+
abortbtn.onclick = function(){
128+
let curRequest = curRequests.pop();
129+
if(curRequest)curRequest.abort();
130+
}
100131
}
101132

102133
function indexDownload(aEles){
103134
if(aEles.length<1)return;
104135
initTxtDownDiv();
136+
rCats=[];
137+
var insertSigns=[];
105138
// var j=0,rCats=[];
106-
var j=0;
139+
var downIndex=0,downNum=0,downOnce=function(){
140+
if(downNum>=aEles.length)return;
141+
let curIndex=downIndex;
142+
let aTag=aEles[curIndex];
143+
let request=(aTag, curIndex)=>{
144+
return [curIndex,GM_xmlhttpRequest({
145+
method: 'GET',
146+
url: aTag.href,
147+
overrideMimeType:"text/html;charset="+document.charset,
148+
onload: function(result) {
149+
var doc = getDocEle(result.responseText);
150+
let nextPage=checkNextPage(doc);
151+
if(nextPage){
152+
nextPage.innerText=aTag.innerText+"\t>>";
153+
aEles.push(nextPage);
154+
let targetIndex = curIndex;
155+
for(let a=0;a<insertSigns.length;a++){
156+
let signs=insertSigns[a],breakSign=false;
157+
if(signs){
158+
for(let b=0;b<signs.length;b++){
159+
let sign=signs[b];
160+
if(sign==curIndex){
161+
targetIndex=a;
162+
breakSign=true;
163+
break;
164+
}
165+
}
166+
}
167+
if(breakSign)break;
168+
}
169+
let insertSign = insertSigns[targetIndex];
170+
if(!insertSign)insertSigns[targetIndex] = [];
171+
insertSigns[targetIndex].push(aEles.length-1);
172+
}
173+
downIndex++;
174+
downNum++;
175+
processDoc(curIndex, aTag, doc);
176+
let request=downOnce();
177+
if(request)curRequests.push(request);
178+
},
179+
onerror: function(e) {
180+
console.warn("error:");
181+
console.log(e);
182+
downIndex++;
183+
downNum++;
184+
processDoc(curIndex, aTag, null);
185+
let request=downOnce();
186+
if(request)curRequests.push(request);
187+
},
188+
onabort: function(e) {
189+
console.warn("abort:");
190+
console.log(e);
191+
downIndex++;
192+
downNum++;
193+
processDoc(curIndex, aTag, null);
194+
let request=downOnce();
195+
if(request)curRequests.push(request);
196+
},
197+
})];
198+
}
199+
if(!aTag){
200+
let waitAtagReadyInterval=setInterval(function(){
201+
if(downNum>=aEles.length)clearInterval(waitAtagReadyInterval);
202+
aTag=aEles[curIndex];
203+
if(aTag){
204+
clearInterval(waitAtagReadyInterval);
205+
request(aTag, curIndex);
206+
}
207+
},1000);
208+
return null;
209+
}
210+
return request(aTag, curIndex);
211+
};
107212
function getDocEle(str){
108213
var doc = null;
109214
try {
@@ -115,18 +220,50 @@
115220
}
116221
return doc;
117222
}
223+
function sortInnerPage(){
224+
var pageArrs=[],maxIndex=0,i,j;
225+
for(i=0;i<insertSigns.length;i++){
226+
var signs=insertSigns[i];
227+
if(signs){
228+
for(j=0;j<signs.length;j++){
229+
var sign=signs[j];
230+
var cat=rCats[sign];
231+
rCats[sign]=null;
232+
if(!pageArrs[i])pageArrs[i]=[];
233+
pageArrs[i].push(cat);
234+
}
235+
}
236+
}
237+
for(i=pageArrs.length-1;i>=0;i--){
238+
let pageArr=pageArrs[i];
239+
if(pageArr){
240+
for(j=pageArr.length-1;j>=0;j--){
241+
rCats.splice(i+1, 0, pageArr[j]);
242+
}
243+
}
244+
}
245+
rCats = rCats.filter(function(e){return e!=null});
246+
}
118247
function processDoc(i, aTag, doc){
119-
j++;
248+
curRequests = curRequests.filter(function(e){return e[0]!=i});
120249
rCats[i]=(aTag.innerText+"\r\n"+getPageContent(doc));
121250
txtDownContent.style.display="block";
122-
txtDownWords.innerHTML=getI18n("downloading",[j,(aEles.length-j),aTag.innerText]);
123-
if(j==aEles.length){
124-
txtDownWords.innerHTML=getI18n("complete",[j]);
251+
txtDownWords.innerHTML=getI18n("downloading",[downNum,(aEles.length-downNum),aTag.innerText]);
252+
if(downNum==aEles.length){
253+
txtDownWords.innerHTML=getI18n("complete",[downNum]);
254+
sortInnerPage();
125255
var blob = new Blob([i18n.info+"\r\n"+document.title+"\r\n\r\n"+rCats.join("\r\n\r\n")], {type: "text/plain;charset=utf-8"});
126256
saveAs(blob, document.title+".txt");
127257
}
128258
}
129-
for(let i=0;i<aEles.length;i++){
259+
for(var i=0;i<downThreadNum;i++){
260+
let request=downOnce();
261+
if(request)curRequests.push(request);
262+
if(downIndex>=aEles.length-1)break;
263+
if(downIndex<downThreadNum-1)downIndex++;
264+
}
265+
266+
/*for(let i=0;i<aEles.length;i++){
130267
let aTag=aEles[i];
131268
GM_xmlhttpRequest({
132269
method: 'GET',
@@ -137,7 +274,19 @@
137274
processDoc(i, aTag, doc);
138275
}
139276
});
277+
}*/
278+
}
279+
280+
function checkNextPage(doc){
281+
let aTags=doc.querySelectorAll("a"),nextPage=null;
282+
for(var i=0;i<aTags.length;i++){
283+
let aTag=aTags[i];
284+
if(innerNextPage.test(aTag.innerText) && /^http/i.test(aTag.href)){
285+
nextPage=aTag;
286+
break;
287+
}
140288
}
289+
return nextPage;
141290
}
142291

143292
function getPageContent(doc){
@@ -271,7 +420,7 @@
271420
break;
272421
}
273422
}
274-
if(!has && aEle.href && /^http/i.test(aEle.href) && /PART\b|Prologue|-\d+|分卷|Chapter\s*\d+|^\s*序\s*$|序\s*言|序\s*章|前\s*言|引\s*言|引\s*子|摘\s*要|楔\s*子|契\s*子|后\s*记|後\s*記|附\s*言|结\s*语|結\s*語|最終話|最终话|[\d|〇|零|一|二|三|四|五|六|七|八|九|十|百|千|万|萬|-]+\s*(、|）|\.\D|章|节|節|回|卷|折|篇|幕|集|话|話)/i.test(aEle.innerText)){
423+
if(!has && aEle.href && /^http/i.test(aEle.href) && (indexReg.test(aEle.innerText) || /chapter[\-_]?\d/.test(aEle.href))){
275424
list.push(aEle);
276425
}
277426
}
@@ -281,11 +430,6 @@
281430
return parseInt(a.innerText.replace(/[^0-9]/ig,"")) - parseInt(b.innerText.replace(/[^0-9]/ig,""));
282431
});
283432
}
284-
if(GM_getValue("contentSortUrl")){
285-
list.sort(function(a,b){
286-
return parseInt(a.href.replace(/[^0-9]/ig,"")) - parseInt(b.href.replace(/[^0-9]/ig,""));
287-
});
288-
}
289433
indexDownload(list);
290434
}else{
291435
var blob = new Blob([i18n.info+"\r\n"+document.title+"\r\n\r\n"+getPageContent(document)], {type: "text/plain;charset=utf-8"});
@@ -302,35 +446,63 @@
302446
var selValue=GM_getValue("selectors");
303447
var selectors=prompt(i18n.del,selValue?selValue:"");
304448
GM_setValue("selectors",selectors);
305-
GM_setValue("contentSort",window.confirm(i18n.reSort));
306-
GM_setValue("contentSortUrl",window.confirm(i18n.reSortUrl));
307-
var urls=window.prompt(i18n.custom,"https://xxx.xxx/book-[20-99].html, https://xxx.xxx/book-[01-10].html");
449+
if(window.confirm(i18n.reSort)){
450+
GM_setValue("contentSort", true);
451+
}else{
452+
GM_setValue("contentSort", false);
453+
}
454+
}
455+
function customDown(){
456+
var urls=window.prompt(i18n.customInfo,"https://xxx.xxx/book-[20-99].html, https://xxx.xxx/book-[01-10].html");
308457
if(urls){
309458
var processEles=[];
310-
[].forEach.call(urls.split(","),function(i){
311-
var varNum=/\[\d+\-\d+\]/.exec(i)[0].trim();
312-
var num1=/\[(\d+)/.exec(varNum)[1].trim();
313-
var num2=/(\d+)\]/.exec(varNum)[1].trim();
314-
var num1Int=parseInt(num1);
315-
var num2Int=parseInt(num2);
316-
var numLen=num1.length;
317-
var needAdd=num1.charAt(0)=="0";
318-
if(num1Int>=num2Int)return;
319-
for(var j=num1Int;j<=num2Int;j++){
320-
var urlIndex=j.toString();
321-
if(needAdd){
322-
while(urlIndex.length<numLen)urlIndex="0"+urlIndex;
459+
if(/^http|^ftp/.test(urls)){
460+
[].forEach.call(urls.split(","),function(i){
461+
var varNum=/\[\d+\-\d+\]/.exec(i)[0].trim();
462+
var num1=/\[(\d+)/.exec(varNum)[1].trim();
463+
var num2=/(\d+)\]/.exec(varNum)[1].trim();
464+
var num1Int=parseInt(num1);
465+
var num2Int=parseInt(num2);
466+
var numLen=num1.length;
467+
var needAdd=num1.charAt(0)=="0";
468+
if(num1Int>=num2Int)return;
469+
for(var j=num1Int;j<=num2Int;j++){
470+
var urlIndex=j.toString();
471+
if(needAdd){
472+
while(urlIndex.length<numLen)urlIndex="0"+urlIndex;
473+
}
474+
var curUrl=i.replace(/\[\d+\-\d+\]/,urlIndex).trim();
475+
var curEle=document.createElement("a");
476+
curEle.href=curUrl;
477+
processEles.push(curEle);
478+
curEle.innerText=processEles.length.toString();
323479
}
324-
var curUrl=i.replace(/\[\d+\-\d+\]/,urlIndex).trim();
325-
var curEle=document.createElement("a");
326-
curEle.href=curUrl;
327-
processEles.push(curEle);
328-
curEle.innerText=processEles.length.toString();
329-
}
330-
});
480+
});
481+
}else{
482+
[].forEach.call(document.querySelectorAll(urls),function(item){
483+
let has=false;
484+
for(var j=0;j<processEles.length;j++){
485+
if(processEles[j].href==item.href){
486+
processEles.splice(j,1);
487+
processEles.push(item);
488+
has=true;
489+
break;
490+
}
491+
}
492+
if(!has && item.href && /^http/i.test(item.href)){
493+
processEles.push(item);
494+
}
495+
});
496+
}
497+
if(GM_getValue("contentSort")){
498+
processEles.sort(function(a,b){
499+
return parseInt(a.innerText.replace(/[^0-9]/ig,"")) - parseInt(b.innerText.replace(/[^0-9]/ig,""));
500+
});
501+
}
331502
indexDownload(processEles);
332503
}
333504
}
334505
GM_registerMenuCommand(i18n.fetch, fetch);
506+
GM_registerMenuCommand(i18n.custom, customDown);
335507
GM_registerMenuCommand(i18n.setting, setDel);
336508
})();

0 commit comments

Comments
 (0)