-
Notifications
You must be signed in to change notification settings - Fork 0
/
config.ini
56 lines (43 loc) · 2.01 KB
/
config.ini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
[Settings]
; Custom User Agent string to mimic browser requests. Default is "AO3 Sample Scraper Bot".
; Example: user_agent = Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3
user_agent =
; The base URL for scraping. This should be the search or tag page on AO3.
; Example: url = https://archiveofourown.org/tags/YourFandom/works
url =
; The page number to start scraping from. If not specified, defaults to 1.
; Example: start_page = 1
start_page =
; The page number to end scraping at. Leave blank to keep open-ended or if using max_work_count.
; Example: end_page = 10
end_page =
; The maximum number of works to scrape. If specified, start_page and end_page are ignored.
; Example: max_work_count = 100
max_work_count =
; The delay (in seconds) between requests to avoid overloading the server. Default is 5 seconds.
; Example: delay = 5
delay =
; The name of the CSV file to save the scraped data. Default is 'scraped_works'.
; Example: csv_file = my_scraped_data
csv_file =
; The path where the CSV file will be saved. Default is the current directory ('./').
; Example: csv_path = /path/to/directory/
csv_path =
; Whether to append to (a) or overwrite (w) the existing CSV file. Default is 'w'.
; Example: file_mode = a
file_mode =
; The delimiter used within the CSV file. Default is '; '.
; Example: internal_delimiter = ;
internal_delimiter =
; Sampling strategy for scraping (null, random, systematic, strata). Leave blank for no sampling.
; Example: sampling_strategy = random
sampling_strategy =
; The percentage of works to sample when using 'random' strategy. Default is 50%.
; Example: sampling_percentage = 25
sampling_percentage =
; The interval of works to sample when using 'systematic' strategy. Example: every 2nd work.
; Example: sampling_n = 2
sampling_n =
; The bins for kudos when using 'strata' sampling, separated by commas.
; Example: kudos_bins = 0, 100, 200, 500
kudos_bins =