1 /** 2 * Handle meta information related to pulling articles from Dev.to 3 */ 4 module devtopull; 5 6 import std.datetime; 7 import std.range; 8 9 import vibe.data.json; 10 11 import devto.api; 12 13 version(unittest) { 14 import unittesting.articlegenerator; 15 import std.algorithm : map, until, equal, fold; 16 } 17 18 /** 19 * Provides information about the last execution to pull. 20 * 21 * This structure will allow retrieving newer articles since the last session. 22 */ 23 struct PullState { 24 /// When the pull was executed 25 SysTime datePulled; 26 /// The newest article ID obtained during the pulling 27 uint lastArticle; 28 } 29 30 /** 31 * Determine if an article is newer than the last PullState. 32 */ 33 @safe pure nothrow @nogc 34 auto isNewerArticle(const ArticleMe am, const PullState ps) { 35 return am.id > ps.lastArticle; 36 } unittest { 37 PullState ps; 38 ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC()); 39 ps.lastArticle = 10_000; 40 alias inPreviousPull = (x) => !x.isNewerArticle(ps); 41 auto devreq = generate!fakeArticleMe 42 .seqence(ps.datePulled) 43 .map!(x => x.deserializeJson!(ArticleMe)) 44 .take(3); 45 assert(devreq.until!inPreviousPull.empty); 46 } unittest { 47 PullState ps; 48 ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC()); 49 ps.lastArticle = 9_999; 50 alias inPreviousPull = (x) => !x.isNewerArticle(ps); 51 auto devreq = generate!fakeArticleMe 52 .seqence(ps.datePulled) 53 .map!(x => x.deserializeJson!(ArticleMe)) 54 .take(3) 55 .map!(x => cast(immutable(ArticleMe))x); 56 assert(devreq.until!inPreviousPull.map!(x => x.id).equal([10_000])); 57 } 58 59 /** 60 * Deserializes file into the PullState structure 61 */ 62 @safe 63 PullState readLastPull(string filename) { 64 import std.file : readText; 65 return deserializeJson!(PullState)(readText(filename)); 66 } 67 68 /** 69 * Serializes PullState structure into file 70 */ 71 @trusted 72 void saveLastPull(const PullState ps, string filename) { 73 import util.file : fileWriter; 74 import std.algorithm : copy; 75 serializeToPrettyJson(ps).copy(fileWriter(filename)); 76 } 77 78 struct DevToRange { 79 private const(ArticleMe)[] function(uint page) @safe articlePage; 80 81 const(ArticleMe)[] data; 82 uint page; 83 84 @safe nothrow pure 85 auto front() { 86 return data.front; 87 } 88 89 @safe nothrow pure 90 auto empty() { 91 return data.empty; 92 } 93 94 @safe pure 95 auto popFront() { 96 data.popFront; 97 } 98 } 99 100 @safe 101 auto devtoMyArticles() { 102 DevToRange dtv; 103 dtv.articlePage = &devArticles; 104 dtv.data = dtv.articlePage(1); 105 return dtv; 106 } 107 108 @safe 109 auto devArticles(uint page) { 110 import vibe.core.log; 111 import vibe.http.client; 112 import std.process; 113 import std.format; 114 115 const(ArticleMe)[] ret; 116 117 requestHTTP(format!"https://dev.to/api/articles/me/all?page=%s"(page), 118 (scope req) @safe { 119 req.headers.addField("api-key", environment["appkey"]); 120 }, 121 (scope res) @safe { 122 logInfo("Response: %d", res.statusCode); 123 if(res.statusCode != 200) 124 return; 125 auto data = res.readJson(); 126 auto savePath = "devArticles"; 127 ret = meArticles(data); 128 } 129 ); 130 131 return ret; 132 }