1 /**
2 	Generates offline documentation in the form of HTML files.
3 
4 	Copyright: © 2012 RejectedSoftware e.K.
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Sönke Ludwig
7 */
8 module ddox.htmlgenerator;
9 
10 import ddox.api;
11 import ddox.entities;
12 import ddox.settings;
13 
14 import std.algorithm : canFind, countUntil, map;
15 import std.array;
16 import std.digest.md;
17 import std.format : formattedWrite;
18 import std.string : startsWith, toLower;
19 import std.traits : EnumMembers;
20 import std.variant;
21 import vibe.core.log;
22 import vibe.core.file;
23 import vibe.core.stream;
24 import vibe.data.json;
25 import vibe.inet.path;
26 import vibe.http.server;
27 import vibe.stream.wrapper : StreamOutputRange;
28 import diet.html;
29 import diet.traits : dietTraits;
30 
31 
32 /*
33 	structure:
34 	/index.html
35 	/pack1/pack2/module1.html
36 	/pack1/pack2/module1/member.html
37 	/pack1/pack2/module1/member.submember.html
38 */
39 
40 version (Windows) version = CaseInsensitiveFS;
41 else version (OSX) version = CaseInsensitiveFS;
42 
43 void generateHtmlDocs(Path dst_path, Package root, GeneratorSettings settings = null)
44 {
45 	import std.algorithm : splitter;
46 	import vibe.web.common : adjustMethodStyle;
47 
48 	if( !settings ) settings = new GeneratorSettings;
49 
50 	version (CaseInsensitiveFS) {
51 		final switch (settings.fileNameStyle) with (MethodStyle) {
52 			case unaltered, camelCase, pascalCase:
53 				logWarn("On Windows and OS X, file names that differ only in their case "
54 					~ "are treated as equal by default. Use one of the "
55 					~ "lower/upper case styles with the --file-name-style "
56 					~ "option to avoid missing files in the generated output.");
57 				break;
58 			case lowerCase, upperCase, lowerUnderscored, upperUnderscored:
59 				break;
60 		}
61 	}
62 
63 	string[string] file_hashes;
64 	string[string] new_file_hashes;
65 
66 	const hash_file_name = dst_path ~ "file_hashes.json";
67 	if (existsFile(hash_file_name)) {
68 		auto hfi = getFileInfo(hash_file_name);
69 		auto hf = readFileUTF8(hash_file_name);
70 		file_hashes = deserializeJson!(string[string])(hf);
71 	}
72 
73 	string linkTo(in Entity ent_, size_t level)
74 	{
75 		import std.typecons : Rebindable;
76 
77 		auto dst = appender!string();
78 		Rebindable!(const(Entity)) ent = ent_;
79 
80 		if( level ) foreach( i; 0 .. level ) dst.put("../");
81 		else dst.put("./");
82 
83 		if( ent !is null ){
84 			if( !ent.parent ){
85 				dst.put("index.html");
86 				return dst.data();
87 			}
88 
89 			Entity nested;
90 			if (
91 				// link parameters to their function
92 				(cast(FunctionDeclaration)ent.parent !is null &&
93 				 (nested = cast(VariableDeclaration)ent) !is null) ||
94 				// link enum members to their enum
95 				(!settings.enumMemberPages &&
96 				 cast(EnumDeclaration)ent.parent !is null &&
97 				 (nested = cast(EnumMemberDeclaration)ent) !is null))
98 				ent = ent.parent;
99 
100 			const(Entity)[] nodes;
101 			size_t mod_idx = 0;
102 			while( ent ){
103 				if( cast(const(Module))ent ) mod_idx = nodes.length;
104 				nodes ~= ent.get;
105 				ent = ent.parent;
106 			}
107 			foreach_reverse(i, n; nodes[mod_idx .. $-1]){
108 				dst.put(n.name[]);
109 				if( i > 0 ) dst.put('/');
110 			}
111 			if( mod_idx == 0 ) dst.put(".html");
112 			else {
113 				dst.put('/');
114 				foreach_reverse(n; nodes[0 .. mod_idx]){
115 					dst.put(adjustMethodStyle(n.name, settings.fileNameStyle));
116 					dst.put('.');
117 				}
118 				dst.put("html");
119 			}
120 
121 			// FIXME: conflicting ids with parameters occurring in multiple overloads
122 			// link nested elements to anchor in parent, e.g. params, enum members
123 			if( nested ){
124 				dst.put('#');
125 				dst.put(nested.name[]);
126 			}
127 		}
128 
129 		return dst.data();
130 	}
131 
132 	void collectChildren(Entity parent, ref DocGroup[][string] pages)
133 	{
134 		Declaration[] members;
135 		if (!settings.enumMemberPages && cast(EnumDeclaration)parent)
136 			return;
137 
138 		if (auto mod = cast(Module)parent) members = mod.members;
139 		else if (auto ctd = cast(CompositeTypeDeclaration)parent) members = ctd.members;
140 		else if (auto td = cast(TemplateDeclaration)parent) members = td.members;
141 
142 		foreach (decl; members) {
143 			if (decl.parent !is parent) continue; // exclude inherited members (issue #120)
144 			auto style = settings.fileNameStyle; // workaround for invalid value when directly used inside lamba
145 			auto name = decl.nestedName.splitter(".").map!(n => adjustMethodStyle(n, style)).join(".");
146 			auto pl = name in pages;
147 			if (pl && !canFind(*pl, decl.docGroup)) *pl ~= decl.docGroup;
148 			else if (!pl) pages[name] = [decl.docGroup];
149 
150 			collectChildren(decl, pages);
151 		}
152 	}
153 
154 	void writeHashedFile(Path filename, scope void delegate(OutputStream) del)
155 	{
156 		import vibe.stream.memory;
157 		assert(filename.startsWith(dst_path));
158 
159 		auto str = createMemoryOutputStream();
160 		del(str);
161 		auto h = md5Of(str.data).toHexString.idup;
162 		auto relfilename = filename[dst_path.length .. $].toString();
163 		auto ph = relfilename in file_hashes;
164 		if (!ph || *ph != h) {
165 			//logInfo("do write %s", filename);
166 			writeFile(filename, str.data);
167 		}
168 		new_file_hashes[relfilename] = h;
169 	}
170 
171 	void visitModule(Module mod, Path pack_path)
172 	{
173 		auto modpath = pack_path ~ PathEntry(mod.name);
174 		if (!existsFile(modpath)) createDirectory(modpath);
175 		logInfo("Generating module: %s", mod.qualifiedName);
176 		writeHashedFile(pack_path ~ PathEntry(mod.name~".html"), (stream) {
177 			generateModulePage(stream, root, mod, settings, ent => linkTo(ent, pack_path.length-dst_path.length));
178 		});
179 
180 		DocGroup[][string] pages;
181 		collectChildren(mod, pages);
182 		foreach (name, decls; pages)
183 			writeHashedFile(modpath ~ PathEntry(name~".html"), (stream) {
184 				generateDeclPage(stream, root, mod, name, decls, settings, ent => linkTo(ent, modpath.length-dst_path.length));
185 			});
186 	}
187 
188 	void visitPackage(Package p, Path path)
189 	{
190 		auto packpath = p.parent ? path ~ PathEntry(p.name) : path;
191 		if( !packpath.empty && !existsFile(packpath) ) createDirectory(packpath);
192 		foreach( sp; p.packages ) visitPackage(sp, packpath);
193 		foreach( m; p.modules ) visitModule(m, packpath);
194 	}
195 
196 	dst_path.normalize();
197 
198 	if( !dst_path.empty && !existsFile(dst_path) ) createDirectory(dst_path);
199 
200 	writeHashedFile(dst_path ~ PathEntry("index.html"), (stream) {
201 		generateApiIndex(stream, root, settings, ent => linkTo(ent, 0));
202 	});
203 
204 	writeHashedFile(dst_path ~ "symbols.js", (stream) {
205 		generateSymbolsJS(stream, root, settings, ent => linkTo(ent, 0));
206 	});
207 
208 	writeHashedFile(dst_path ~ PathEntry("sitemap.xml"), (stream) {
209 		generateSitemap(stream, root, settings, ent => linkTo(ent, 0));
210 	});
211 
212 	visitPackage(root, dst_path);
213 
214 	// delete obsolete files
215 	foreach (f; file_hashes.byKey)
216 		if (f !in new_file_hashes) {
217 			try removeFile(dst_path ~ Path(f));
218 			catch (Exception e) logWarn("Failed to remove obsolete file '%s': %s", f, e.msg);
219 		}
220 
221 	// write new file hash list
222 	writeFileUTF8(hash_file_name, new_file_hashes.serializeToJsonString());
223 }
224 
225 class DocPageInfo {
226 	string delegate(in Entity ent) linkTo;
227 	GeneratorSettings settings;
228 	Package rootPackage;
229 	Entity node;
230 	Module mod;
231 	DocGroup[] docGroups; // for multiple doc groups with the same name
232 	string nestedName;
233 
234 	// mysql-native hack
235 	@property auto defaultMacros()  { import ddox.ddoc; return s_defaultMacros;  }
236 	@property auto overrideMacros() { import ddox.ddoc; return s_overrideMacros; }
237 
238 	@property NavigationType navigationType() const { return settings.navigationType; }
239 	string formatType(CachedType tp, bool include_code_tags = true) { return .formatType(tp, linkTo, include_code_tags); }
240 	void renderTemplateArgs(R)(R output, Declaration decl) { .renderTemplateArgs(output, decl, linkTo); }
241 	string formatDoc(DocGroup group, int hlevel, bool delegate(string) display_section)
242 	{
243 		if (!group) return null;
244 		// TODO: memoize the DocGroupContext
245 		return group.comment.renderSections(new DocGroupContext(group, linkTo, settings), display_section, hlevel);
246 	}
247 }
248 
249 @dietTraits
250 struct DdoxDietTraits(HTMLOutputStyle htmlStyle) {
251 	// fields and functions must be static atm., see https://github.com/rejectedsoftware/diet-ng/issues/33
252 	enum HTMLOutputStyle htmlOutputStyle = htmlStyle;
253 }
254 
255 void generateSitemap(OutputStream dst, Package root_package, GeneratorSettings settings, string delegate(in Entity) link_to, HTTPServerRequest req = null)
256 {
257 	dst.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
258 	dst.write("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
259 
260 	void writeEntry(string[] parts...){
261 		dst.write("<url><loc>");
262 		foreach( p; parts )
263 			dst.write(p);
264 		dst.write("</loc></url>\n");
265 	}
266 
267 	void writeEntityRec(Entity ent){
268 		import std.string;
269 		if( !cast(Package)ent || ent is root_package ){
270 			auto link = link_to(ent);
271 			if( indexOf(link, '#') < 0 ) // ignore URLs with anchors
272 				writeEntry((settings.siteUrl ~ Path(link)).toString());
273 		}
274 		ent.iterateChildren((ch){ writeEntityRec(ch); return true; });
275 	}
276 
277 	writeEntityRec(root_package);
278 
279 	dst.write("</urlset>\n");
280 	dst.flush();
281 }
282 
283 void generateSymbolsJS(OutputStream dst, Package root_package, GeneratorSettings settings, string delegate(in Entity) link_to)
284 {
285 	import std.typecons : Tuple, tuple;
286 
287 	bool[Tuple!(Entity, CachedString)] visited;
288 
289 	auto rng = StreamOutputRange(dst);
290 
291 	void writeEntry(Entity ent) {
292 		auto key = tuple(ent.parent, ent.name);
293 		if (cast(Package)ent || cast(TemplateParameterDeclaration)ent) return;
294 		if (key in visited) return;
295 		visited[key] = true;
296 
297 		string kind = ent.classinfo.name.split(".")[$-1].toLower;
298 		const(CachedString)[] cattributes;
299 		if (auto fdecl = cast(FunctionDeclaration)ent) cattributes = fdecl.attributes;
300 		else if (auto adecl = cast(AliasDeclaration)ent) cattributes = adecl.attributes;
301 		else if (auto tdecl = cast(TypedDeclaration)ent) cattributes = tdecl.type.attributes;
302 		auto attributes = cattributes.map!(a => a.str.startsWith("@") ? a[1 .. $] : a);
303 		(&rng).formattedWrite(`{name: '%s', kind: "%s", path: '%s', attributes: %s},`, ent.qualifiedName, kind, link_to(ent), attributes);
304 		rng.put('\n');
305 	}
306 
307 	void writeEntryRec(Entity ent) {
308 		writeEntry(ent);
309 		if (cast(FunctionDeclaration)ent) return;
310 		ent.iterateChildren((ch) { writeEntryRec(ch); return true; });
311 	}
312 
313 	rng.put("// symbol index generated by DDOX - do not edit\n");
314 	rng.put("var symbols = [\n");
315 	writeEntryRec(root_package);
316 	rng.put("];\n");
317 }
318 
319 void generateApiIndex(OutputStream dst, Package root_package, GeneratorSettings settings, string delegate(in Entity) link_to, HTTPServerRequest req = null)
320 {
321 	auto info = new DocPageInfo;
322 	info.linkTo = link_to;
323 	info.settings = settings;
324 	info.rootPackage = root_package;
325 	info.node = root_package;
326 
327 	auto rng = StreamOutputRange(dst);
328 	final switch (settings.htmlOutputStyle)
329 	{
330 		foreach (htmlOutputStyle; EnumMembers!HTMLOutputStyle)
331 		case htmlOutputStyle:
332 		{
333 			rng.compileHTMLDietFile!("ddox.overview.dt", req, info, DdoxDietTraits!(htmlOutputStyle));
334 			return;
335 		}
336 	}
337 }
338 
339 void generateModulePage(OutputStream dst, Package root_package, Module mod, GeneratorSettings settings, string delegate(in Entity) link_to, HTTPServerRequest req = null)
340 {
341 	auto info = new DocPageInfo;
342 	info.linkTo = link_to;
343 	info.settings = settings;
344 	info.rootPackage = root_package;
345 	info.mod = mod;
346 	info.node = mod;
347 	info.docGroups = null;
348 
349 	auto rng = StreamOutputRange(dst);
350 	final switch (settings.htmlOutputStyle)
351 	{
352 		foreach (htmlOutputStyle; EnumMembers!HTMLOutputStyle)
353 		case htmlOutputStyle:
354 		{
355 			rng.compileHTMLDietFile!("ddox.module.dt", req, info, DdoxDietTraits!(htmlOutputStyle));
356 			return;
357 		}
358 	}
359 }
360 
361 void generateDeclPage(OutputStream dst, Package root_package, Module mod, string nested_name, DocGroup[] docgroups, GeneratorSettings settings, string delegate(in Entity) link_to, HTTPServerRequest req = null)
362 {
363 	import std.algorithm : sort;
364 
365 	auto info = new DocPageInfo;
366 	info.linkTo = link_to;
367 	info.settings = settings;
368 	info.rootPackage = root_package;
369 	info.mod = mod;
370 	info.node = mod;
371 	info.docGroups = docgroups;//docGroups(mod.lookupAll!Declaration(nested_name));
372 	sort!((a, b) => cmpKind(a.members[0], b.members[0]))(info.docGroups);
373 	info.nestedName = nested_name;
374 
375 	auto rng = StreamOutputRange(dst);
376 	final switch (settings.htmlOutputStyle)
377 	{
378 		foreach (htmlOutputStyle; EnumMembers!HTMLOutputStyle)
379 		case htmlOutputStyle:
380 		{
381 			rng.compileHTMLDietFile!("ddox.docpage.dt", req, info, DdoxDietTraits!(htmlOutputStyle));
382 			return;
383 		}
384 	}
385 }
386 
387 private bool cmpKind(in Entity a, in Entity b)
388 {
389 	static immutable kinds = [
390 		DeclarationKind.Variable,
391 		DeclarationKind.Function,
392 		DeclarationKind.Struct,
393 		DeclarationKind.Union,
394 		DeclarationKind.Class,
395 		DeclarationKind.Interface,
396 		DeclarationKind.Enum,
397 		DeclarationKind.EnumMember,
398 		DeclarationKind.Template,
399 		DeclarationKind.TemplateParameter,
400 		DeclarationKind.Alias
401 	];
402 
403 	auto ad = cast(const(Declaration))a;
404 	auto bd = cast(const(Declaration))b;
405 
406 	if (!ad && !bd) return false;
407 	if (!ad) return false;
408 	if (!bd) return true;
409 
410 	auto ak = kinds.countUntil(ad.kind);
411 	auto bk = kinds.countUntil(bd.kind);
412 
413 	if (ak < 0 && bk < 0) return false;
414 	if (ak < 0) return false;
415 	if (bk < 0) return true;
416 
417 	return ak < bk;
418 }