1 /**
2  * Handle meta information related to pulling articles from Dev.to
3  */
4 module devtopull;
5 
6 import std.datetime;
7 import std.range;
8 
9 import vibe.data.json;
10 
11 import devto.api;
12 
13 version(unittest) {
14     import unittesting.articlegenerator;
15     import std.algorithm : map, until, equal, fold;
16 }
17 
18 /**
19  * Provides information about the last execution to pull.
20  *
21  * This structure will allow retrieving newer articles since the last session.
22  */
23 struct PullState {
24     /// When the pull was executed
25     SysTime datePulled;
26     /// The newest article ID obtained during the pulling
27     uint lastArticle;
28 }
29 
30 /**
31  * Determine if an article is newer than the last PullState.
32  */
33 @safe pure nothrow @nogc
34 auto isNewerArticle(const ArticleMe am, const PullState ps) {
35     return am.id > ps.lastArticle;
36 } unittest {
37     PullState ps;
38     ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC());
39     ps.lastArticle = 10_000;
40     alias inPreviousPull = (x) => !x.isNewerArticle(ps);
41     auto devreq = generate!fakeArticleMe
42         .seqence(ps.datePulled)
43         .map!(x => x.deserializeJson!(ArticleMe))
44         .take(3);
45     assert(devreq.until!inPreviousPull.empty);
46 } unittest {
47     PullState ps;
48     ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC());
49     ps.lastArticle = 9_999;
50     alias inPreviousPull = (x) => !x.isNewerArticle(ps);
51     auto devreq = generate!fakeArticleMe
52         .seqence(ps.datePulled)
53         .map!(x => x.deserializeJson!(ArticleMe))
54         .take(3)
55         .map!(x => cast(immutable(ArticleMe))x);
56     assert(devreq.until!inPreviousPull.map!(x => x.id).equal([10_000]));
57 }
58 
59 /**
60  * Deserializes file into the PullState structure
61  */
62 @safe
63 PullState readLastPull(string filename) {
64     import std.file : readText;
65     return deserializeJson!(PullState)(readText(filename));
66 }
67 
68 /**
69  * Serializes PullState structure into file
70  */
71 @trusted
72 void saveLastPull(const PullState ps, string filename) {
73     import util.file : fileWriter;
74     import std.algorithm : copy;
75     serializeToPrettyJson(ps).copy(fileWriter(filename));
76 }
77 
78 struct DevToRange {
79     private const(ArticleMe)[] function(uint page) @safe articlePage;
80 
81     const(ArticleMe)[] data;
82     uint page;
83 
84     @safe nothrow pure
85     auto front() {
86         return data.front;
87     }
88 
89     @safe nothrow pure
90     auto empty() {
91         return data.empty;
92     }
93 
94     @safe pure
95     auto popFront() {
96         data.popFront;
97     }
98 }
99 
100 @safe
101 auto devtoMyArticles() {
102     DevToRange dtv;
103     dtv.articlePage = &devArticles;
104     dtv.data = dtv.articlePage(1);
105     return dtv;
106 }
107 
108 @safe
109 auto devArticles(uint page) {
110     import vibe.core.log;
111     import vibe.http.client;
112     import std.process;
113     import std.format;
114 
115     const(ArticleMe)[] ret;
116 
117     requestHTTP(format!"https://dev.to/api/articles/me/all?page=%s"(page),
118         (scope req) @safe {
119             req.headers.addField("api-key", environment["appkey"]);
120         },
121         (scope res) @safe {
122             logInfo("Response: %d", res.statusCode);
123             if(res.statusCode != 200)
124                 return;
125             auto data = res.readJson();
126             auto savePath = "devArticles";
127             ret = meArticles(data);
128         }
129     );
130 
131     return ret;
132 }