1 /** 2 * This module focuses on methods for writing articles to disk. 3 * 4 * Article storage is broken into two parts, meta data and markdown content. 5 * A folder location may be specified but markdown content will always be stored 6 * in a sub folder `articleContentFolder`. 7 */ 8 module articlewriter; 9 10 import vibe.core.log; 11 import vibe.data.json; 12 13 import std; 14 15 import devto.api; 16 17 version(unittest) { 18 import unittesting.articlegenerator; 19 import std.algorithm : map, until, equal, fold; 20 import devtopull; 21 } 22 23 /** 24 * This is the structure used to store article meta data as Json to disk. 25 * 26 * This content is pimarily defined from the devto API, where `content_file` 27 * is a separate field used to define the relative filesystem location of 28 * the markdown file to the meta file. 29 */ 30 struct ArticleMeta { 31 @optional string title; 32 @optional bool published; 33 @optional int id; 34 @optional int organization_id; 35 @optional string canonical_url; 36 @optional string description; 37 @optional string main_image; // Cover image 38 @optional string series; 39 @optional string type_of; 40 @optional const(string)[] tags; 41 @optional string content_file; 42 @optional string slug; 43 44 /** 45 * Construct a Meta article from `ArticleMe`. 46 * 47 * Useful when processing existing articles from the devto API. 48 */ 49 @safe pure nothrow @nogc 50 this(const ArticleMe me) { 51 canonical_url = me.canonical_url; 52 description = me.description; 53 id = me.id; 54 main_image = me.cover_image; 55 published = me.published; 56 series = ""; 57 tags = me.tag_list; 58 title = me.title; 59 type_of = me.type_of; 60 slug = me.slug; 61 } 62 63 /** 64 * Convert Meta data into `ArticleCreate`. 65 * 66 * Useful to call devto api to create an article. 67 * 68 * This does not retrieve the `content_file` from disk, it will be require 69 * to add the `body_markdown` after making this conversion. 70 */ 71 @safe pure nothrow @nogc 72 T opCast(T : ArticleCreate)() { 73 ArticleCreate ac; 74 ac.title = title; 75 ac.published = published; 76 ac.series = series; 77 ac.main_image = main_image; 78 ac.canonical_url = canonical_url; 79 ac.description = description; 80 ac.tags = tags; 81 ac.organization_id = organization_id; 82 ac.slug = slug; 83 return ac; 84 } 85 } 86 87 /// This defines the sub folder for storing articles. 88 enum articleContentFolder = "Articles"; 89 90 /** 91 * Separate the Meta data and Content for easier disk writing and retrieval. 92 */ 93 alias Article = Tuple!(ArticleMeta, "meta", string, "content"); 94 95 /** 96 * Given a range of `ArticleMe` separate the data into meta data and content. 97 * 98 * Returns: Range of `Article` 99 */ 100 @safe pure nothrow 101 auto structureArticles(Range)(Range articles) if (isInputRange!Range 102 && is(Unqual!(ElementType!Range) == ArticleMe)) { 103 return articles.map!(x => Article(ArticleMeta(x), x.body_markdown)); 104 } unittest { 105 auto exampleResponse = `[ 106 { 107 "published_at":null, 108 "canonical_url":"http://example.com/", 109 "comments_count":0, 110 "page_views_count":0, 111 "id":12345, 112 "published":false, 113 "body_markdown":"Content Body", 114 "title":"Article Title", 115 "description":"", 116 "tag_list":[ 117 "testing" 118 ], 119 "type_of":"article", 120 "slug":"slugger" 121 }]`.parseJsonString.meArticles; 122 123 ArticleMeta ans; 124 ans.type_of = "article"; 125 ans.id = 12345; 126 ans.title = "Article Title"; 127 ans.description = ""; 128 ans.main_image = ""; 129 ans.published = false; 130 ans.tags = ["testing"]; 131 ans.canonical_url = "http://example.com/"; 132 ans.series = ""; 133 ans.slug = "slugger"; 134 135 import std.algorithm; 136 import std.conv; 137 138 assert(structureArticles(exampleResponse).equal([Article(ans, "Content Body")])); 139 } 140 141 /** 142 * Meta information about pathing for article content and meta data. 143 */ 144 private struct Paths { 145 /// Location article content/markdown is stored. 146 string articleSaveLocation; 147 /// Location meta data is stored. 148 string metaSaveLocation; 149 /// Article content location relative to metaSaveLocation, 150 /// stored in meta data. 151 string articleMetaPath; 152 } 153 154 /* 155 * Creates a path relationship for an article. 156 */ 157 @safe pure nothrow 158 private const(Paths) articlePaths(string folder, const ArticleMeta am) 159 out(path) { 160 assert(path.articleSaveLocation != path.metaSaveLocation, "Can't save to same file"); 161 assert(path.articleSaveLocation.endsWith(path.articleMetaPath), "Meta Reference to Article File must match."); 162 assert(!path.articleSaveLocation.find(folder).empty, "Save Locations should utilized specified folder"); 163 assert(!path.metaSaveLocation.find(folder).empty, "Save Locations should utilized specified folder"); 164 } do { 165 Paths ret; 166 167 auto filename = am.slug; 168 assert(isValidFilename(filename), "Article Name Conflicts with valid file name."); 169 170 ret.articleMetaPath = buildPath(articleContentFolder, filename.setExtension("md")); 171 ret.articleSaveLocation = folder.buildPath(ret.articleMetaPath); 172 ret.metaSaveLocation = folder.buildPath(filename.setExtension("devto")); 173 174 return ret; 175 } unittest { 176 ArticleMeta meta; 177 meta.title = "Article Title"; 178 meta.slug = "article_title_oul3o"; 179 auto suppressWarning = articlePaths("someLocation", meta); 180 } 181 182 /** 183 * Writes articles to specified folder. 184 * 185 * Article content will be stored in `articleContentFolder` as a sub folder. 186 * 187 * Params: 188 * articles = Build from call to `structureArticles()` 189 * folder = Filesystem folder location to place articles 190 * 191 * See_Also: 192 * structureArticles, Article, articleContentFolder 193 */ 194 @trusted 195 void saveArticle(Article article, string folder) { 196 import util.file : fileWriter; 197 import std.algorithm : copy; 198 auto paths = articlePaths(folder, article.meta); 199 200 article.meta.content_file = paths.articleMetaPath; 201 article.content.copy(fileWriter(paths.articleSaveLocation)); 202 article.meta.serializeToPrettyJson.copy(fileWriter(paths.metaSaveLocation)); 203 } 204 205 /** 206 * Reduces the range into the newest article. 207 * 208 * Combine this with std.range.tee to write out articles as 209 * the latest article is being reduced. 210 */ 211 @safe 212 auto reduceToNewest(Range)(Range articles) 213 if(is(ElementType!Range == Article)) { 214 return articles.fold!((a, b) => a.meta.id < b.meta.id ? b : a); 215 } unittest { 216 PullState ps; 217 ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC()); 218 alias inPreviousPull = (x) => !x.isNewerArticle(ps); 219 auto devreq = generate!fakeArticleMe 220 .seqence(ps.datePulled + dur!"days"(1)) 221 .map!(x => x.deserializeJson!(ArticleMe)) 222 .take(3); 223 auto newestid = devreq.front.id; 224 assert(devreq.until!inPreviousPull 225 .structureArticles 226 .reduceToNewest 227 .meta.id == newestid); 228 }