1 module install;
2 
3 import std.algorithm;
4 import std.array;
5 import std.exception;
6 import std.file;
7 import std.path;
8 import std.string;
9 
10 import ae.sys.file;
11 import ae.utils.array;
12 import ae.utils.json;
13 
14 import common;
15 import config : config;
16 import custom;
17 
18 static import std.process;
19 
20 alias copy = std.file.copy; // https://issues.dlang.org/show_bug.cgi?id=14817
21 
22 version (Windows)
23 {
24 	enum string binExt = ".exe";
25 	enum string dmdConfigName = "sc.ini";
26 }
27 else
28 {
29 	enum string binExt = "";
30 	enum string dmdConfigName = "dmd.conf";
31 }
32 
33 version (Posix)
34 {
35 	import core.sys.posix.unistd;
36 	import std.conv : octal;
37 }
38 
39 version (Windows)
40 	enum string platformDir = "windows";
41 else
42 version (linux)
43 	enum string platformDir = "linux";
44 else
45 version (OSX)
46 	enum string platformDir = "osx";
47 else
48 version (FreeBSD)
49 	enum string platformDir = "freebsd";
50 else
51 	enum string platformDir = null;
52 
53 string[] findDMD()
54 {
55 	string[] result;
56 	foreach (pathEntry; std.process.environment.get("PATH", null).split(pathSeparator))
57 	{
58 		auto dmd = pathEntry.buildPath("dmd" ~ binExt);
59 		if (dmd.exists)
60 			result ~= dmd;
61 	}
62 	return result;
63 }
64 
65 string selectInstallPath(string location)
66 {
67     string[] candidates;
68     if (location)
69 	{
70 		auto dmd = location.absolutePath();
71 		@property bool ok() { return dmd.exists && dmd.isFile; }
72 
73 		if (!ok)
74 		{
75 			string newDir;
76 
77 			bool dirOK(string dir)
78 			{
79 				newDir = dmd.buildPath(dir);
80 				return newDir.exists && newDir.isDir;
81 			}
82 
83 			bool tryDir(string dir)
84 			{
85 				if (dirOK(dir))
86 				{
87 					dmd = newDir;
88 					return true;
89 				}
90 				return false;
91 			}
92 
93 			tryDir("dmd2");
94 
95 			static if (platformDir)
96 				tryDir(platformDir);
97 
98 			enforce(!dirOK("bin32") || !dirOK("bin64"),
99 				"Ambiguous model in path - please specify full path to DMD binary");
100 			tryDir("bin") || tryDir("bin32") || tryDir("bin64");
101 
102 			dmd = dmd.buildPath("dmd" ~ binExt);
103 			enforce(ok, "DMD installation not detected at " ~ location);
104 		}
105 
106 		candidates = [dmd];
107 	}
108 	else
109 	{
110 		candidates = findDMD();
111 		enforce(candidates.length, "DMD not found in PATH - " ~
112 			"add DMD to PATH or specify install location explicitly");
113 	}
114 
115 	foreach (candidate; candidates)
116 	{
117 		if (candidate.buildNormalizedPath.startsWith(config.local.workDir.buildNormalizedPath))
118 		{
119 			log("Skipping DMD installation under Digger workDir (" ~ config.local.workDir ~ "): " ~ candidate);
120 			continue;
121 		}
122 
123 		log("Found DMD executable: " ~ candidate);
124 		return candidate;
125 	}
126 
127 	throw new Exception("No suitable DMD installation found.");
128 }
129 
130 string findConfig(string dmdPath)
131 {
132 	string configPath;
133 
134 	bool pathOK(string path)
135 	{
136 		configPath = path.buildPath(dmdConfigName);
137 		return configPath.exists;
138 	}
139 
140 	import std.process : execute;
141 	auto result = execute([dmdPath, "--help"]);
142 	if (result.status == 0)
143 		foreach (line; result.output.splitLines)
144 			if (line.skipOver("Config file: "))
145 				if (line.exists)
146 					return line;
147 
148 	if (pathOK(dmdPath.dirName))
149 		return configPath;
150 
151 	auto home = std.process.environment.get("HOME", null);
152 	if (home && pathOK(home))
153 		return configPath;
154 
155 	version (Posix)
156 		if (pathOK("/etc/"))
157 			return configPath;
158 
159 	throw new Exception(format("Can't find DMD configuration file %s " ~
160 		"corresponding to DMD located at %s", dmdConfigName, dmdPath));
161 }
162 
163 struct ComponentPaths
164 {
165 	string binPath;
166 	string libPath;
167 	string phobosPath;
168 	string druntimePath;
169 }
170 
171 string commonModel(ref BuildInfo buildInfo)
172 {
173 	enforce(buildInfo.config.components.common.models.length == 1, "Multi-model install is not yet supported");
174 	return buildInfo.config.components.common.models[0];
175 }
176 
177 ComponentPaths parseConfig(string dmdPath, BuildInfo buildInfo)
178 {
179 	auto configPath = findConfig(dmdPath);
180 	log("Found DMD configuration: " ~ configPath);
181 
182 	string[string] vars = std.process.environment.toAA();
183 	bool parsing = false;
184 	foreach (line; configPath.readText().splitLines())
185 	{
186 		if (line.startsWith("[") && line.endsWith("]"))
187 		{
188 			auto sectionName = line[1..$-1];
189 			parsing = sectionName == "Environment"
190 			       || sectionName == "Environment" ~ buildInfo.commonModel;
191 		}
192 		else
193 		if (parsing && line.canFind("="))
194 		{
195 			string name, value;
196 			list(name, null, value) = line.findSplit("=");
197 			auto parts = value.split("%");
198 			for (size_t n = 1; n < parts.length; n+=2)
199 				if (!parts[n].length)
200 					parts[n] = "%";
201 				else
202 				if (parts[n] == "@P")
203 					parts[n] = configPath.dirName();
204 				else
205 					parts[n] = vars.get(parts[n], parts[n]);
206 			value = parts.join();
207 			vars[name] = value;
208 		}
209 	}
210 
211 	string[] parseParameters(string s, char escape = '\\', char separator = ' ')
212 	{
213 		string[] result;
214 		while (s.length)
215 			if (s[0] == separator)
216 				s = s[1..$];
217 			else
218 			{
219 				string p;
220 				if (s[0] == '"')
221 				{
222 					s = s[1..$];
223 					bool escaping, end;
224 					while (s.length && !end)
225 					{
226 						auto c = s[0];
227 						s = s[1..$];
228 						if (!escaping && c == '"')
229 							end = true;
230 						else
231 						if (!escaping && c == escape)
232 							escaping = true;
233 						else
234 						{
235 							if (escaping && c != escape)
236 								p ~= escape;
237 							p ~= c;
238 							escaping = false;
239 						}
240 					}
241 				}
242 				else
243 					list(p, null, s) = s.findSplit([separator]);
244 				result ~= p;
245 			}
246 		return result;
247 	}
248 
249 	string[] dflags = parseParameters(vars.get("DFLAGS", null));
250 	string[] importPaths = dflags
251 		.filter!(s => s.startsWith("-I"))
252 		.map!(s => s[2..$].split(";"))
253 		.join();
254 
255 	version (Windows)
256 		string[] libPaths = parseParameters(vars.get("LIB", null), 0, ';');
257 	else
258 		string[] libPaths = dflags
259 			.filter!(s => s.startsWith("-L-L"))
260 			.map!(s => s[4..$])
261 			.array();
262 
263 	string findPath(string[] paths, string name, string[] testFiles)
264 	{
265 		auto results = paths.find!(path => testFiles.any!(testFile => path.buildPath(testFile).exists));
266 		enforce(!results.empty, "Can't find %s (%-(%s or %)). Looked in: %s".format(name, testFiles, paths));
267 		auto result = results.front.buildNormalizedPath();
268 		auto testFile = testFiles.find!(testFile => result.buildPath(testFile).exists).front;
269 		log("Found %s (%s): %s".format(name, testFile, result));
270 		return result;
271 	}
272 
273 	ComponentPaths result;
274 	result.binPath = dmdPath.dirName();
275 	result.libPath = findPath(libPaths, "Phobos static library", [getLibFileName(buildInfo)]);
276 	result.phobosPath = findPath(importPaths, "Phobos source code", ["std/stdio.d"]);
277 	result.druntimePath = findPath(importPaths, "Druntime import files", ["object.d", "object.di"]);
278 	return result;
279 }
280 
281 string getLibFileName(BuildInfo buildInfo)
282 {
283 	version (Windows)
284 	{
285 		auto model = buildInfo.commonModel;
286 		return "phobos%s.lib".format(model == "32" ? "" : model);
287 	}
288 	else
289 		return "libphobos2.a";
290 }
291 
292 struct InstalledObject
293 {
294 	/// File name in backup directory
295 	string name;
296 
297 	/// Original location.
298 	/// Path is relative to uninstall.json's directory.
299 	string path;
300 
301 	/// MD5 sum of the NEW object's contents
302 	/// (not the one in the install directory).
303 	/// For directories, this is the MD5 sum
304 	/// of all files sorted by name (see mdDir).
305 	string hash;
306 }
307 
308 struct UninstallData
309 {
310 	InstalledObject*[] objects;
311 }
312 
313 void install(bool yes, bool dryRun, string location = null)
314 {
315 	assert(!yes || !dryRun, "Mutually exclusive options");
316 	auto dmdPath = selectInstallPath(location);
317 
318 	auto buildInfoPath = resultDir.buildPath(buildInfoFileName);
319 	enforce(buildInfoPath.exists,
320 		buildInfoPath ~ " not found - please purge cache and rebuild");
321 	auto buildInfo = buildInfoPath.readText().jsonParse!BuildInfo();
322 
323 	auto componentPaths = parseConfig(dmdPath, buildInfo);
324 
325 	auto verb = dryRun ? "Would" : "Will";
326 	log("%s install:".format(verb));
327 	log(" - Binaries to:           " ~ componentPaths.binPath);
328 	log(" - Libraries to:          " ~ componentPaths.libPath);
329 	log(" - Phobos source code to: " ~ componentPaths.phobosPath);
330 	log(" - Druntime includes to:  " ~ componentPaths.druntimePath);
331 
332 	auto uninstallPath = buildPath(componentPaths.binPath, ".digger-install");
333 	auto uninstallFileName = buildPath(uninstallPath, "uninstall.json");
334 	bool updating = uninstallFileName.exists;
335 	if (updating)
336 	{
337 		log("Found previous installation data in " ~ uninstallPath);
338 		log("%s update existing installation.".format(verb));
339 	}
340 	else
341 	{
342 		log("This %s be a new Digger installation.".format(verb.toLower));
343 		log("Backups and uninstall data %s be saved in %s".format(verb.toLower, uninstallPath));
344 	}
345 
346 	auto libFileName = getLibFileName(buildInfo);
347 	auto libName = libFileName.stripExtension ~ "-" ~ buildInfo.commonModel ~ libFileName.extension;
348 
349 	static struct Item
350 	{
351 		string name, srcPath, dstPath;
352 	}
353 
354 	Item[] items =
355 	[
356 		Item("dmd"  ~ binExt, buildPath(resultDir, "bin", "dmd"  ~ binExt), dmdPath),
357 		Item("rdmd" ~ binExt, buildPath(resultDir, "bin", "rdmd" ~ binExt), buildPath(componentPaths.binPath, "rdmd" ~ binExt)),
358 		Item(libName        , buildPath(resultDir, "lib", libFileName)    , buildPath(componentPaths.libPath, libFileName)),
359 		Item("object.di"    , buildPath(resultDir, "import", "object.{d,di}").globFind,
360 		                                                                    buildPath(componentPaths.druntimePath, "object.{d,di}").globFind),
361 		Item("core"         , buildPath(resultDir, "import", "core")      , buildPath(componentPaths.druntimePath, "core")),
362 		Item("std"          , buildPath(resultDir, "import", "std")       , buildPath(componentPaths.phobosPath, "std")),
363 		Item("etc"          , buildPath(resultDir, "import", "etc")       , buildPath(componentPaths.phobosPath, "etc")),
364 	];
365 
366 	InstalledObject*[string] existingComponents;
367 	bool[string] updateNeeded;
368 
369 	UninstallData uninstallData;
370 
371 	if (updating)
372 	{
373 		uninstallData = uninstallFileName.readText.jsonParse!UninstallData;
374 		foreach (obj; uninstallData.objects)
375 			existingComponents[obj.name] = obj;
376 	}
377 
378 	log("Preparing object list...");
379 
380 	foreach (item; items)
381 	{
382 		log(" - " ~ item.name);
383 
384 		auto obj = new InstalledObject(item.name, item.dstPath.relativePath(uninstallPath), mdObject(item.srcPath));
385 		auto pexistingComponent = item.name in existingComponents;
386 		if (pexistingComponent)
387 		{
388 			auto existingComponent = *pexistingComponent;
389 
390 			enforce(existingComponent.path == obj.path,
391 				"Updated component has a different path (%s vs %s), aborting."
392 				.format(existingComponents[item.name].path, obj.path));
393 
394 			verifyObject(existingComponent, uninstallPath, "update");
395 
396 			updateNeeded[item.name] = existingComponent.hash != obj.hash;
397 			existingComponent.hash = obj.hash;
398 		}
399 		else
400 			uninstallData.objects ~= obj;
401 	}
402 
403 	log("Testing write access and filesystem boundaries:");
404 
405 	string[] dirs = items.map!(item => item.dstPath.dirName).array.sort().uniq().array;
406 	foreach (dir; dirs)
407 	{
408 		log(" - %s".format(dir));
409 		auto testPathA = dir.buildPath(".digger-test");
410 		auto testPathB = componentPaths.binPath.buildPath(".digger-test2");
411 
412 		std.file.write(testPathA, "test");
413 		{
414 			scope(failure) remove(testPathA);
415 			rename(testPathA, testPathB);
416 		}
417 		remove(testPathB);
418 	}
419 
420 	version (Posix)
421 	{
422 		int owner = dmdPath.getOwner();
423 		int group = dmdPath.getGroup();
424 		int mode = items.front.dstPath.getAttributes() & octal!666;
425 		log("UID=%d GID=%d Mode=%03o".format(owner, group, mode));
426 	}
427 
428 	log("Things to do:");
429 
430 	foreach (item; items)
431 	{
432 		enforce(item.srcPath.exists, "Can't find source for component %s: %s".format(item.name, item.srcPath));
433 		enforce(item.dstPath.exists, "Can't find target for component %s: %s".format(item.name, item.dstPath));
434 
435 		string action;
436 		if (updating && item.name in existingComponents)
437 			action = updateNeeded[item.name] ? "Update" : "Skip unchanged";
438 		else
439 			action = "Install";
440 		log(" - %s component %s from %s to %s".format(action, item.name, item.srcPath, item.dstPath));
441 	}
442 
443 	log("You %s be able to undo this action by running `digger uninstall`.".format(verb.toLower));
444 
445 	if (dryRun)
446 	{
447 		log("Dry run, exiting.");
448 		return;
449 	}
450 
451 	if (yes)
452 		log("Proceeding with installation.");
453 	else
454 	{
455 		import std.stdio : stdin, stderr;
456 
457 		string result;
458 		do
459 		{
460 			stderr.write("Proceed with installation? [Y/n] "); stderr.flush();
461 			result = stdin.readln().chomp().toLower();
462 		} while (result != "y" && result != "n" && result != "");
463 		if (result == "n")
464 			return;
465 	}
466 
467 	enforce(updating || !uninstallPath.exists, "Uninstallation directory exists without uninstall.json: " ~ uninstallPath);
468 
469 	log("Saving uninstall information...");
470 
471 	if (!updating)
472 		mkdir(uninstallPath);
473 	std.file.write(uninstallFileName, toJson(uninstallData));
474 
475 	log("Backing up original files...");
476 
477 	foreach (item; items)
478 		if (item.name !in existingComponents)
479 		{
480 			log(" - " ~ item.name);
481 			auto backupPath = buildPath(uninstallPath, item.name);
482 			rename(item.dstPath, backupPath);
483 		}
484 
485 	if (updating)
486 	{
487 		log("Cleaning up existing Digger-installed files...");
488 
489 		foreach (item; items)
490 			if (item.name in existingComponents && updateNeeded[item.name])
491 			{
492 				log(" - " ~ item.name);
493 				rmObject(item.dstPath);
494 			}
495 	}
496 
497 	log("Installing new files...");
498 
499 	foreach (item; items)
500 		if (item.name !in existingComponents || updateNeeded[item.name])
501 		{
502 			log(" - " ~ item.name);
503 			atomic!cpObject(item.srcPath, item.dstPath);
504 		}
505 
506 	version (Posix)
507 	{
508 		log("Applying attributes...");
509 
510 		bool isRoot = geteuid()==0;
511 
512 		foreach (item; items)
513 			if (item.name !in existingComponents || updateNeeded[item.name])
514 			{
515 				log(" - " ~ item.name);
516 				item.dstPath.recursive!setMode(mode);
517 
518 				if (isRoot)
519 					item.dstPath.recursive!setOwner(owner, group);
520 				else
521 					if (item.dstPath.getOwner() != owner || item.dstPath.getGroup() != group)
522 						log("Warning: UID/GID mismatch for " ~ item.dstPath);
523 			}
524 	}
525 
526 	log("Install OK.");
527 	log("You can undo this action by running `digger uninstall`.");
528 }
529 
530 void uninstall(bool dryRun, bool force, string location = null)
531 {
532 	string uninstallPath;
533 	if (location.canFind(".digger-install"))
534 		uninstallPath = location;
535 	else
536 	{
537 		auto dmdPath = selectInstallPath(location);
538 		auto binPath = dmdPath.dirName();
539 		uninstallPath = buildPath(binPath, ".digger-install");
540 	}
541 	auto uninstallFileName = buildPath(uninstallPath, "uninstall.json");
542 	enforce(uninstallFileName.exists, "Can't find uninstallation data: " ~ uninstallFileName);
543 	auto uninstallData = uninstallFileName.readText.jsonParse!UninstallData;
544 
545 	if (!force)
546 	{
547 		log("Verifying files to be uninstalled...");
548 
549 		foreach (obj; uninstallData.objects)
550 			verifyObject(obj, uninstallPath, "uninstall");
551 
552 		log("Verify OK.");
553 	}
554 
555 	log(dryRun ? "Actions to run:" : "Uninstalling...");
556 
557 	void runAction(void delegate() action)
558 	{
559 		if (!force)
560 			action();
561 		else
562 			try
563 				action();
564 			catch (Exception e)
565 				log("Ignoring error: " ~ e.msg);
566 	}
567 
568 	void uninstallObject(InstalledObject* obj)
569 	{
570 		auto src = buildNormalizedPath(uninstallPath, obj.name);
571 		auto dst = buildNormalizedPath(uninstallPath, obj.path);
572 
573 		if (!src.exists) // --force
574 		{
575 			log(" - %s component %s with no backup".format(dryRun ? "Would skip" : "Skipping", obj.name));
576 			return;
577 		}
578 
579 		if (dryRun)
580 		{
581 			log(" - Would remove " ~ dst);
582 			log("   Would move " ~ src ~ " to " ~ dst);
583 		}
584 		else
585 		{
586 			log(" - Removing " ~ dst);
587 			runAction({ rmObject(dst); });
588 			log("   Moving " ~ src ~ " to " ~ dst);
589 			runAction({ rename(src, dst); });
590 		}
591 	}
592 
593 	foreach (obj; uninstallData.objects)
594 		runAction({ uninstallObject(obj); });
595 
596 	if (dryRun)
597 		return;
598 
599 	remove(uninstallFileName);
600 
601 	if (!force)
602 		rmdir(uninstallPath); // should be empty now
603 	else
604 		rmdirRecurse(uninstallPath);
605 
606 	log("Uninstall OK.");
607 }
608 
609 string globFind(string path)
610 {
611 	auto results = dirEntries(path.dirName, path.baseName, SpanMode.shallow);
612 	enforce(!results.empty, "Can't find: " ~ path);
613 	auto result = results.front;
614 	results.popFront();
615 	enforce(results.empty, "Multiple matches: " ~ path);
616 	return result;
617 }
618 
619 void verifyObject(InstalledObject* obj, string uninstallPath, string verb)
620 {
621 	auto path = buildNormalizedPath(uninstallPath, obj.path);
622 	enforce(path.exists, "Can't find item to %s: %s".format(verb, path));
623 	auto hash = mdObject(path);
624 	enforce(hash == obj.hash,
625 		"Object changed since it was installed: %s\nPlease %s manually.".format(path, verb));
626 }
627 
628 version(Posix) bool attrIsExec(int attr) { return (attr & octal!111) != 0; }
629 
630 /// Set access modes while preserving executable bit.
631 version(Posix)
632 void setMode(string fn, int mode)
633 {
634 	auto attr = fn.getAttributes();
635 	mode |= attr & ~octal!777;
636 	if (attr.attrIsExec || attr.attrIsDir)
637 		mode = mode | ((mode & octal!444) >> 2); // executable iff readable
638 	fn.setAttributes(mode);
639 }
640 
641 /// Apply a function recursively to all files and directories under given path.
642 template recursive(alias fun)
643 {
644 	void recursive(Args...)(string fn, auto ref Args args)
645 	{
646 		fun(fn, args);
647 		if (fn.isDir)
648 			foreach (de; fn.dirEntries(SpanMode.shallow))
649 				recursive(de.name, args);
650 	}
651 }
652 
653 void rmObject(string path) { path.isDir ? path.rmdirRecurse() : path.remove(); }
654 
655 void cpObject(string src, string dst)
656 {
657 	if (src.isDir)
658 	{
659 		mkdir(dst);
660 		dst.setAttributes(src.getAttributes());
661 		foreach (de; src.dirEntries(SpanMode.shallow))
662 			cpObject(de.name, buildPath(dst, de.baseName));
663 	}
664 	else
665 	{
666 		src.copy(dst);
667 		dst.setAttributes(src.getAttributes());
668 	}
669 }
670 
671 string mdDir(string dir)
672 {
673 	import std.stdio : File;
674 	import std.digest.md;
675 
676 	auto dataChunks = dir
677 		.dirEntries(SpanMode.breadth)
678 		.filter!(de => de.isFile)
679 		.map!(de => de.name.replace(`\`, `/`))
680 		.array()
681 		.sort()
682 		.map!(name => File(name, "rb").byChunk(4096))
683 		.joiner();
684 
685 	MD5 digest;
686 	digest.start();
687 	foreach (chunk; dataChunks)
688 		digest.put(chunk);
689 	auto result = digest.finish();
690 	// https://issues.dlang.org/show_bug.cgi?id=9279
691 	auto str = result.toHexString();
692 	return str[].idup;
693 }
694 
695 string mdObject(string path)
696 {
697 	static if (__VERSION__ < 2080)
698 		import std.digest.digest : toHexString;
699 	else
700 		import std.digest : toHexString;
701 
702 	if (path.isDir)
703 		return path.mdDir();
704 	else
705 	{
706 		auto result = path.mdFile();
707 		// https://issues.dlang.org/show_bug.cgi?id=9279
708 		auto str = result.toHexString();
709 		return str[].idup;
710 	}
711 }