1 /** 2 * This module focuses on methods for writing articles to disk. 3 * 4 * Article storage is broken into two parts, meta data and markdown content. 5 * A folder location may be specified but markdown content will always be stored 6 * in a sub folder `articleContentFolder`. 7 */ 8 module articlewriter; 9 10 import vibe.core.log; 11 import vibe.data.json; 12 13 import std; 14 15 import devto.api; 16 17 /** 18 * This is the structure used to store article meta data as Json to disk. 19 * 20 * This content is pimarily defined from the devto API, where `content_file` 21 * is a separate field used to define the relative filesystem location of 22 * the markdown file to the meta file. 23 */ 24 struct ArticleMeta { 25 @optional string title; 26 @optional bool published; 27 @optional int id; 28 @optional int organization_id; 29 @optional string canonical_url; 30 @optional string description; 31 @optional string main_image; // Cover image 32 @optional string series; 33 @optional string type_of; 34 @optional string[] tags; 35 @optional string content_file; 36 @optional string slug; 37 38 /** 39 * Construct a Meta article from `ArticleMe`. 40 * 41 * Useful when processing existing articles from the devto API. 42 */ 43 @safe pure nothrow @nogc 44 this(ArticleMe me) { 45 canonical_url = me.canonical_url; 46 description = me.description; 47 id = me.id; 48 main_image = me.cover_image; 49 published = me.published; 50 series = ""; 51 tags = me.tag_list; 52 title = me.title; 53 type_of = me.type_of; 54 slug = me.slug; 55 } 56 57 /** 58 * Convert Meta data into `ArticleCreate`. 59 * 60 * Useful to call devto api to create an article. 61 * 62 * This does not retrieve the `content_file` from disk, it will be require 63 * to add the `body_markdown` after making this conversion. 64 */ 65 @safe pure nothrow @nogc 66 T opCast(T : ArticleCreate)() { 67 ArticleCreate ac; 68 ac.title = title; 69 ac.published = published; 70 ac.series = series; 71 ac.main_image = main_image; 72 ac.canonical_url = canonical_url; 73 ac.description = description; 74 ac.tags = tags; 75 ac.organization_id = organization_id; 76 ac.slug = slug; 77 return ac; 78 } 79 } 80 81 /// This defines the sub folder for storing articles. 82 enum articleContentFolder = "Articles"; 83 84 /** 85 * Separate the Meta data and Content for easier disk writing and retrieval. 86 */ 87 alias Article = Tuple!(ArticleMeta, "meta", string, "content"); 88 89 /** 90 * Given a range of `ArticleMe` separate the data into meta data and content. 91 * 92 * Returns: Range of `Article` 93 */ 94 @safe pure nothrow 95 auto structureArticles(Range)(Range articles) if (isInputRange!Range 96 && is(ElementType!Range == ArticleMe)) { 97 return articles.map!(x => Article(ArticleMeta(x), x.body_markdown)); 98 } unittest { 99 auto exampleResponse = `[ 100 { 101 "published_at":null, 102 "canonical_url":"http://example.com/", 103 "comments_count":0, 104 "page_views_count":0, 105 "id":12345, 106 "published":false, 107 "body_markdown":"Content Body", 108 "title":"Article Title", 109 "description":"", 110 "tag_list":[ 111 "testing" 112 ], 113 "type_of":"article", 114 "slug":"slugger" 115 }]`.parseJsonString.meArticles; 116 117 ArticleMeta ans; 118 ans.type_of = "article"; 119 ans.id = 12345; 120 ans.title = "Article Title"; 121 ans.description = ""; 122 ans.main_image = ""; 123 ans.published = false; 124 ans.tags = ["testing"]; 125 ans.canonical_url = "http://example.com/"; 126 ans.series = ""; 127 ans.slug = "slugger"; 128 129 import std.algorithm; 130 import std.conv; 131 132 assert(structureArticles(exampleResponse).equal([Article(ans, "Content Body")])); 133 } 134 135 /* 136 * Creates an iopipe output range for characters to the filename specified. 137 */ 138 @trusted 139 private auto fileWriter(string filename) { 140 import iopipe.valve; 141 import iopipe.textpipe; 142 import iopipe.bufpipe; 143 import std.io; 144 145 return bufd!char 146 .push!(p => p 147 .encodeText 148 .outputPipe(std.io.File(filename, mode!"w").refCounted)) 149 .textOutput; 150 } 151 152 /** 153 * Meta information about pathing for article content and meta data. 154 */ 155 private struct Paths { 156 /// Location article content/markdown is stored. 157 string articleSaveLocation; 158 /// Location meta data is stored. 159 string metaSaveLocation; 160 /// Article content location relative to metaSaveLocation, 161 /// stored in meta data. 162 string articleMetaPath; 163 } 164 165 /* 166 * Creates a path relationship for an article. 167 */ 168 @safe pure nothrow 169 private Paths articlePaths(string folder, ArticleMeta am) 170 out(path) { 171 assert(path.articleSaveLocation != path.metaSaveLocation, "Can't save to same file"); 172 assert(path.articleSaveLocation.endsWith(path.articleMetaPath), "Meta Reference to Article File must match."); 173 assert(!path.articleSaveLocation.find(folder).empty, "Save Locations should utilized specified folder"); 174 assert(!path.metaSaveLocation.find(folder).empty, "Save Locations should utilized specified folder"); 175 } do { 176 Paths ret; 177 178 auto filename = am.slug; 179 assert(isValidFilename(filename), "Article Name Conflicts with valid file name."); 180 181 ret.articleMetaPath = buildPath(articleContentFolder, filename.setExtension("md")); 182 ret.articleSaveLocation = folder.buildPath(ret.articleMetaPath); 183 ret.metaSaveLocation = folder.buildPath(filename.setExtension("devto")); 184 185 return ret; 186 } unittest { 187 ArticleMeta meta; 188 meta.title = "Article Title"; 189 meta.slug = "article_title_oul3o"; 190 articlePaths("someLocation", meta); 191 } 192 193 /** 194 * Writes articles to specified folder. 195 * 196 * Article content will be stored in `articleContentFolder` as a sub folder. 197 * 198 * Params: 199 * articles = Build from call to `structureArticles()` 200 * folder = Filesystem folder location to place articles 201 * 202 * See Also: 203 * structureArticles, Article, articleContentFolder 204 */ 205 @safe 206 void saveArticles(Range)(Range articles, string folder) if(is(ElementType!Range == Article)) 207 in(buildPath(folder,articleContentFolder).exists, format("Create direcorty `buildPath(\"%s\",articleContentFolder)` before calling to save articles.", folder)) 208 { 209 @trusted 210 void saveArticle(Article article) { 211 import std.algorithm : copy; 212 auto paths = articlePaths(folder, article.meta); 213 214 article.meta.content_file = paths.articleMetaPath; 215 article.content.copy(fileWriter(paths.articleSaveLocation)); 216 article.meta.serializeToPrettyJson.copy(fileWriter(paths.metaSaveLocation)); 217 } 218 219 articles.each!saveArticle; 220 221 }