1 /**
2  * This module focuses on methods for writing articles to disk.
3  *
4  * Article storage is broken into two parts, meta data and markdown content.
5  * A folder location may be specified but markdown content will always be stored
6  * in a sub folder `articleContentFolder`.
7  */
8 module articlewriter;
9 
10 import vibe.core.log;
11 import vibe.data.json;
12 
13 import std;
14 
15 import devto.api;
16 
17 version(unittest) {
18     import unittesting.articlegenerator;
19     import std.algorithm : map, until, equal, fold;
20     import devtopull;
21 }
22 
23 /**
24  * This is the structure used to store article meta data as Json to disk.
25  *
26  * This content is pimarily defined from the devto API, where `content_file`
27  * is a separate field used to define the relative filesystem location of
28  * the markdown file to the meta file.
29  */
30 struct ArticleMeta {
31     @optional string title;
32     @optional bool published;
33     @optional int id;
34     @optional int organization_id;
35     @optional string canonical_url;
36     @optional string description;
37     @optional string main_image; // Cover image
38     @optional string series;
39     @optional string type_of;
40     @optional const(string)[] tags;
41     @optional string content_file;
42     @optional string slug;
43 
44     /**
45      * Construct a Meta article from `ArticleMe`.
46      *
47      * Useful when processing existing articles from the devto API.
48      */
49     @safe pure nothrow @nogc
50     this(const ArticleMe me) {
51         canonical_url = me.canonical_url;
52         description = me.description;
53         id = me.id;
54         main_image = me.cover_image;
55         published = me.published;
56         series = "";
57         tags = me.tag_list;
58         title = me.title;
59         type_of = me.type_of;
60         slug = me.slug;
61     }
62 
63     /**
64      * Convert Meta data into `ArticleCreate`.
65      *
66      * Useful to call devto api to create an article.
67      *
68      * This does not retrieve the `content_file` from disk, it will be require
69      * to add the `body_markdown` after making this conversion.
70      */
71     @safe pure nothrow @nogc
72     T opCast(T : ArticleCreate)() {
73         ArticleCreate ac;
74         ac.title = title;
75         ac.published = published;
76         ac.series = series;
77         ac.main_image = main_image;
78         ac.canonical_url = canonical_url;
79         ac.description = description;
80         ac.tags = tags;
81         ac.organization_id = organization_id;
82         ac.slug = slug;
83         return ac;
84     }
85 }
86 
87 /// This defines the sub folder for storing articles.
88 enum articleContentFolder = "Articles";
89 
90 /**
91  * Separate the Meta data and Content for easier disk writing and retrieval.
92  */
93 alias Article = Tuple!(ArticleMeta, "meta", string, "content");
94 
95 /**
96  * Given a range of `ArticleMe` separate the data into meta data and content.
97  *
98  * Returns: Range of `Article`
99  */
100 @safe pure nothrow
101 auto structureArticles(Range)(Range articles) if (isInputRange!Range
102                                && is(Unqual!(ElementType!Range) == ArticleMe)) {
103     return articles.map!(x => Article(ArticleMeta(x), x.body_markdown));
104 } unittest {
105     auto exampleResponse = `[
106   {
107     "published_at":null,
108     "canonical_url":"http://example.com/",
109     "comments_count":0,
110     "page_views_count":0,
111     "id":12345,
112     "published":false,
113     "body_markdown":"Content Body",
114     "title":"Article Title",
115     "description":"",
116     "tag_list":[
117       "testing"
118     ],
119     "type_of":"article",
120     "slug":"slugger"
121   }]`.parseJsonString.meArticles;
122 
123     ArticleMeta ans;
124     ans.type_of = "article";
125     ans.id = 12345;
126     ans.title = "Article Title";
127     ans.description = "";
128     ans.main_image = "";
129     ans.published = false;
130     ans.tags = ["testing"];
131     ans.canonical_url = "http://example.com/";
132     ans.series = "";
133     ans.slug = "slugger";
134 
135     import std.algorithm;
136     import std.conv;
137 
138     assert(structureArticles(exampleResponse).equal([Article(ans, "Content Body")]));
139 }
140 
141 /**
142  * Meta information about pathing for article content and meta data.
143  */
144 private struct Paths {
145     /// Location article content/markdown is stored.
146     string articleSaveLocation;
147     /// Location meta data is stored.
148     string metaSaveLocation;
149     /// Article content location relative to metaSaveLocation,
150     /// stored in meta data.
151     string articleMetaPath;
152 }
153 
154 /*
155  * Creates a path relationship for an article.
156  */
157 @safe pure nothrow
158 private const(Paths) articlePaths(string folder, const ArticleMeta am)
159 out(path) {
160     assert(path.articleSaveLocation != path.metaSaveLocation, "Can't save to same file");
161     assert(path.articleSaveLocation.endsWith(path.articleMetaPath), "Meta Reference to Article File must match.");
162     assert(!path.articleSaveLocation.find(folder).empty, "Save Locations should utilized specified folder");
163     assert(!path.metaSaveLocation.find(folder).empty, "Save Locations should utilized specified folder");
164 } do {
165     Paths ret;
166 
167     auto filename = am.slug;
168     assert(isValidFilename(filename), "Article Name Conflicts with valid file name.");
169 
170     ret.articleMetaPath = buildPath(articleContentFolder, filename.setExtension("md"));
171     ret.articleSaveLocation = folder.buildPath(ret.articleMetaPath);
172     ret.metaSaveLocation = folder.buildPath(filename.setExtension("devto"));
173 
174     return ret;
175 } unittest {
176     ArticleMeta meta;
177     meta.title = "Article Title";
178     meta.slug = "article_title_oul3o";
179     auto suppressWarning = articlePaths("someLocation", meta);
180 }
181 
182 /**
183  * Writes articles to specified folder.
184  *
185  * Article content will be stored in `articleContentFolder` as a sub folder.
186  *
187  * Params:
188  *     articles = Build from call to `structureArticles()`
189  *     folder   = Filesystem folder location to place articles
190  *
191  * See_Also:
192  *     structureArticles, Article, articleContentFolder
193  */
194 @trusted
195 void saveArticle(Article article, string folder) {
196     import util.file : fileWriter;
197     import std.algorithm : copy;
198     auto paths = articlePaths(folder, article.meta);
199 
200     article.meta.content_file = paths.articleMetaPath;
201     article.content.copy(fileWriter(paths.articleSaveLocation));
202     article.meta.serializeToPrettyJson.copy(fileWriter(paths.metaSaveLocation));
203 }
204 
205 /**
206  * Reduces the range into the newest article.
207  *
208  * Combine this with std.range.tee to write out articles as
209  * the latest article is being reduced.
210  */
211 @safe
212 auto reduceToNewest(Range)(Range articles)
213                                         if(is(ElementType!Range == Article)) {
214     return articles.fold!((a, b) => a.meta.id < b.meta.id ? b : a);
215 } unittest {
216     PullState ps;
217     ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC());
218     alias inPreviousPull = (x) => !x.isNewerArticle(ps);
219     auto devreq = generate!fakeArticleMe
220         .seqence(ps.datePulled + dur!"days"(1))
221         .map!(x => x.deserializeJson!(ArticleMe))
222         .take(3);
223     auto newestid = devreq.front.id;
224     assert(devreq.until!inPreviousPull
225            .structureArticles
226            .reduceToNewest
227            .meta.id == newestid);
228 }