You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

scan-adoc-refs 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use Getopt::Long;
  5. use IO::File;
  6. use JSON;
  7. use Data::Dumper;
  8. my $generate_depends;
  9. GetOptions("depends=s" => \$generate_depends) or
  10. die("Error in command line arguments\n");
  11. my $environments = {
  12. default => 1,
  13. wiki => 1,
  14. manvolnum => 1,
  15. pvelogo => 0, # ignore
  16. };
  17. my $fileinfo = {};
  18. my $start_env = [];
  19. foreach my $e (keys %$environments) {
  20. push @$start_env, $e if $environments->{$e};
  21. }
  22. my $env_stack = [$start_env];
  23. my $env_name_stack = [];
  24. sub reset_environment_stack {
  25. $env_stack = [$start_env];
  26. $env_name_stack = [];
  27. }
  28. sub push_environment {
  29. my ($env, $not) = @_;
  30. die "undefined environment '$env'\n" if !defined($environments->{$env});
  31. # FIXME: this seems wrong (nested env?)?
  32. return if !$environments->{$env}; # do not track
  33. if ($not) {
  34. my $new_env = [];
  35. foreach my $e (@{$env_stack->[-1]}) {
  36. if ($e ne $env) {
  37. push @$new_env, $e;
  38. }
  39. }
  40. die "empty environment" if !scalar($new_env);
  41. push @$env_stack, $new_env;
  42. } else {
  43. push @$env_stack, [$env];
  44. }
  45. push @$env_name_stack, $env;
  46. }
  47. sub pop_environment {
  48. my ($env) = @_;
  49. die "undefined environment '$env'\n" if !defined($environments->{$env});
  50. return if !$environments->{$env}; # do not track
  51. pop @$env_stack;
  52. my $res = pop @$env_name_stack;
  53. die "environment missmatch ($res != $env)\n" if $res ne $env;
  54. }
  55. sub register_include {
  56. my ($filename, $include_filename, $env_list) = @_;
  57. foreach my $e (@$env_list) {
  58. $fileinfo->{include}->{$e}->{$filename}->{$include_filename} = 1;
  59. }
  60. }
  61. sub register_blockid {
  62. my ($filename, $blockid, $reftext, $env_list) = @_;
  63. foreach my $e (@$env_list) {
  64. my $fn = $fileinfo->{blockid}->{$e}->{$blockid};
  65. die "blockid '$blockid' already defined in $fn"
  66. if defined($fn);
  67. $fileinfo->{blockid}->{$e}->{$blockid} = $filename;
  68. $fileinfo->{reftext}->{$e}->{$blockid} = $reftext
  69. if defined($reftext);
  70. }
  71. }
  72. sub register_title {
  73. my ($filename, $env, $doctype, $title, $blockid) = @_;
  74. # fixme: what about other macros?
  75. $title =~ s/\{pve\}/Proxmox VE/g;
  76. $title =~ s!http://\S+\[(.*?)\]!$1!g;
  77. # register document title (onyl once)
  78. if (!defined($fileinfo->{titles}->{$env}->{$filename})) {
  79. $fileinfo->{titles}->{$env}->{$filename} = $title;
  80. if (defined($doctype)) {
  81. $fileinfo->{doctype}->{$env}->{$filename} = $doctype;
  82. } else {
  83. die "unable to change title (no doctype)"
  84. if !defined($fileinfo->{doctype}->{$env}->{$filename});
  85. }
  86. } elsif (!defined($doctype)) {
  87. # change title via :title: attribute
  88. $fileinfo->{titles}->{$env}->{$filename} = $title;
  89. }
  90. if (defined($doctype) && ($env eq 'manvolnum') && ($doctype == 0)) {
  91. if ($title =~ m/.*\(([1-8])\)\s*$/) {
  92. $fileinfo->{mansection}->{$env}->{$filename} = $1;
  93. }
  94. }
  95. if ($blockid) {
  96. die "internal error"
  97. if !defined($fileinfo->{blockid}->{$env}->{$blockid});
  98. $fileinfo->{reftitle}->{$env}->{$blockid} = $title;
  99. }
  100. }
  101. sub scan_adoc_file {
  102. my ($filename) = @_;
  103. reset_environment_stack();
  104. # print "SCAN $filename\n";
  105. my $fh = IO::File->new("$filename", "r") or
  106. die "unable to open file '$filename' - $!\n";
  107. my $env_last_line = {};
  108. my $env_last_blockid = {};
  109. while (defined (my $line = <$fh>)) {
  110. if ($line =~ m/^if(n?)def::(\S+)\[(.*)\]\s*$/) {
  111. my ($not, $env, $text) = ($1, $2, $3);
  112. die "unsuported ifdef usage - implement me" if $text;
  113. push_environment($env, $not);
  114. next;
  115. } elsif ($line =~ m/^endif::(\S+)\[(.*)\]\s*$/) {
  116. my ($env, $text) = ($1, $2);
  117. die "unsuported ifdef usage - implement me" if $text;
  118. pop_environment($env);
  119. next;
  120. } elsif ($line =~ m/^include::(\S+)\[.*\]\s*$/) {
  121. register_include($filename, $1, $env_stack->[-1]);
  122. next;
  123. }
  124. # try to detect titles
  125. foreach my $e (@{$env_stack->[-1]}) {
  126. if ($line =~ m/^===+$/) {
  127. register_title($filename, $e, 0, $env_last_line->{$e},
  128. $env_last_blockid->{$e});
  129. } elsif ($line =~ m/^---+$/) {
  130. register_title($filename, $e, 1, $env_last_line->{$e},
  131. $env_last_blockid->{$e});
  132. } elsif ($line =~ m/^~~~+$/) {
  133. register_title($filename, $e, 2, $env_last_line->{$e},
  134. $env_last_blockid->{$e});
  135. } elsif ($line =~ m/^\^\^\^+$/) {
  136. register_title($filename, $e, 3, $env_last_line->{$e},
  137. $env_last_blockid->{$e});
  138. } elsif ($line =~ m/^= +(\S.*?)( +=)?$/) {
  139. register_title($filename, $e, 0, $1, $env_last_blockid->{$e});
  140. } elsif ($line =~ m/^== +(\S.*?)( +==)?$/) {
  141. register_title($filename, $e, 1, $1, $env_last_blockid->{$e});
  142. } elsif ($line =~ m/^=== +(\S.*?)( +===)?$/) {
  143. register_title($filename, $e, 2, $1, $env_last_blockid->{$e});
  144. } elsif ($line =~ m/^==== +(\S.*?)( +====)?$/) {
  145. register_title($filename, $e, 3, $1, $env_last_blockid->{$e});
  146. }
  147. $env_last_line->{$e} = $line;
  148. chomp $env_last_line->{$e};
  149. }
  150. if ($line =~ m/^:(\S+?):\s*(.*\S)?\s*$/) {
  151. my ($key, $value) = ($1, $2);
  152. if ($key eq 'pve-toplevel') {
  153. foreach my $e (@{$env_stack->[-1]}) {
  154. my $title = $fileinfo->{titles}->{$e}->{$filename};
  155. die "not title for toplevel file '$filename' (env=$e)\n"
  156. if !defined($title);
  157. $fileinfo->{toplevel}->{$e}->{$filename} = 1;
  158. }
  159. } elsif ($key eq 'title') {
  160. foreach my $e (@{$env_stack->[-1]}) {
  161. register_title($filename, $e, undef, $value);
  162. }
  163. }
  164. }
  165. if ($line =~ m/^\[\[(.*)\]\]\s*$/) {
  166. my $blockid = $1;
  167. die "implement me" if $blockid =~m/,/;
  168. my $reftext = '';
  169. register_blockid($filename, $blockid, $reftext, $env_stack->[-1]);
  170. foreach my $e (@{$env_stack->[-1]}) {
  171. $env_last_blockid->{$e} = $blockid;
  172. }
  173. }
  174. if ($line =~ m/^\s*$/) {
  175. foreach my $e (@{$env_stack->[-1]}) {
  176. delete $env_last_blockid->{$e};
  177. }
  178. }
  179. # fixme: "anchor:"
  180. # bibliography anchors
  181. if ($line =~ m/\[\[\[([^\]]*)\]\]\]/) {
  182. my $blockid = $1;
  183. die "implement me" if $blockid =~m/,/;
  184. register_blockid($filename, $blockid, "&#91;$blockid&#93;", $env_stack->[-1]);
  185. }
  186. }
  187. }
  188. my $scanned_files = {};
  189. while (my $filename = shift) {
  190. next if $filename !~ m/\.adoc$/; # skip attributes.txt
  191. next if $filename =~ m/-(opts|synopsis)\.adoc$/;
  192. next if $scanned_files->{$filename};
  193. scan_adoc_file($filename);
  194. $scanned_files->{$filename} = 1;
  195. }
  196. sub resolve_link_target {
  197. my ($env, $filename) = @_;
  198. my $include_hash = $fileinfo->{include}->{$env};
  199. my $repeat = 1;
  200. while ($repeat) {
  201. $repeat = 0;
  202. foreach my $fn (keys %$include_hash) {
  203. if ($include_hash->{$fn}->{$filename}) {
  204. next if ($fn eq 'pve-admin-guide.adoc') &&
  205. $fileinfo->{outfile}->{$env}->{$filename};
  206. $filename = $fn;
  207. $repeat = 1;
  208. last;
  209. }
  210. }
  211. }
  212. return $filename;
  213. }
  214. # try to generate output file mapping
  215. foreach my $e (@$start_env) {
  216. my $toplevel_hash = $fileinfo->{toplevel}->{$e};
  217. foreach my $fn (sort keys %$toplevel_hash) {
  218. my $mansection = $fileinfo->{mansection}->{manvolnum}->{$fn};
  219. if ($e eq 'wiki') {
  220. my $realfn = $fn;
  221. $realfn =~ s/\.adoc$//;
  222. if (defined($mansection) && ($mansection == 5)) {
  223. $realfn .= ".$mansection";
  224. }
  225. $realfn = "$realfn-plain.html";
  226. $fileinfo->{outfile}->{$e}->{$fn} = $realfn;
  227. } elsif ($e eq 'manvolnum') {
  228. my $realfn = $fn;
  229. $realfn =~ s/\.adoc$//;
  230. die "toplevel file '$fn' is not marked as manual page!" if !$mansection;
  231. $realfn .= ".$mansection";
  232. $fileinfo->{outfile}->{$e}->{$fn} = $realfn;
  233. } elsif ($e eq 'default') {
  234. my $realfn = $fn;
  235. $realfn =~ s/\.adoc$//;
  236. if (defined($mansection) && ($mansection == 5)) {
  237. $realfn .= ".$mansection";
  238. $realfn = "$realfn.html";
  239. } else {
  240. if (($fn ne 'pve-admin-guide.adoc') &&
  241. $fileinfo->{doctype}->{$e}->{$fn} == 0) {
  242. $realfn = "chapter-$realfn.html";
  243. } else {
  244. $realfn = "$realfn.html";
  245. }
  246. }
  247. $fileinfo->{outfile}->{$e}->{$fn} = $realfn;
  248. }
  249. }
  250. }
  251. # now resolve blockids
  252. foreach my $e (@$start_env) {
  253. my $blockid_hash = $fileinfo->{blockid}->{$e};
  254. foreach my $blockid (keys %$blockid_hash) {
  255. my $fn = resolve_link_target($e, $blockid_hash->{$blockid});
  256. if ($e eq 'wiki') {
  257. my $title = $fileinfo->{titles}->{$e}->{$fn};
  258. $title =~ s/\s/_/g;
  259. die "found not title for '$fn' in env '$e'" if !$title;
  260. $fileinfo->{blockid_target}->{$e}->{$blockid} = "link:/wiki/$title#$blockid";
  261. # we do not produce wiki pages for all content
  262. #my $realfn = $fileinfo->{outfile}->{$e}->{$fn};
  263. #warn "no output file mapping for '$fn' ($e)\n" if !$realfn;
  264. } elsif ($e eq 'default') {
  265. my $realfn = $fileinfo->{outfile}->{$e}->{$fn} ||
  266. die "no output file mapping for '$fn'\n";
  267. $fileinfo->{blockid_target}->{$e}->{$blockid} = "link:/pve-docs/$realfn#$blockid";
  268. } elsif ($e eq 'manvolnum') {
  269. # we do not produce manpages for all content
  270. # my $realfn = $fileinfo->{outfile}->{$e}->{$fn} ||
  271. # warn "no output file mapping for '$fn'\n";
  272. $fileinfo->{blockid_target}->{$e}->{$blockid} = $fn;
  273. }
  274. }
  275. }
  276. my $makevar_hash = {};
  277. sub makevar_define {
  278. my ($varname) = @_;
  279. die "makefile variable '$varname' already defined\n"
  280. if exists($makevar_hash->{$varname});
  281. $makevar_hash->{$varname} = {};
  282. }
  283. sub makevar_list_add {
  284. my ($varname, $value) = @_;
  285. die "makefile variable '$varname' not defined\n"
  286. if !exists($makevar_hash->{$varname});
  287. $makevar_hash->{$varname}->{$value} = 1;
  288. }
  289. sub makevar_dump {
  290. my $txt = '';
  291. foreach my $varname (sort keys %$makevar_hash) {
  292. $txt .= "$varname =";
  293. foreach my $value (sort keys %{$makevar_hash->{$varname}}) {
  294. $txt .= " \\\n\t$value";
  295. }
  296. $txt .= "\n\n";
  297. }
  298. return $txt;
  299. }
  300. if ($generate_depends) {
  301. my $tmpfilename = "${generate_depends}.tmp";
  302. my $outfh = IO::File->new($tmpfilename, "w") ||
  303. die "unable to open temporary file '$tmpfilename'\n";
  304. my $depends = {};
  305. foreach my $e (@$start_env) {
  306. my $env_data = $fileinfo->{include}->{$e};
  307. my $add_depends;
  308. $add_depends = sub {
  309. my ($fn, $dep) = @_;
  310. $depends->{$fn}->{$dep} = 1;
  311. foreach my $nd (keys %{$env_data->{$dep}}) {
  312. &$add_depends($fn, $nd);
  313. }
  314. };
  315. foreach my $fn (keys %$env_data) {
  316. foreach my $dep (keys %{$env_data->{$fn}}) {
  317. &$add_depends($fn, $dep);
  318. }
  319. }
  320. }
  321. my $depend_varname_hash = {};
  322. foreach my $fn (sort keys %$depends) {
  323. my $basename = uc($fn);
  324. $basename =~s/\.adoc$//i;
  325. $basename =~s/[^A-Za-z0-9]/_/g;
  326. my $varname1 = "${basename}_ADOCSOURCES";
  327. my $varname2 = "${basename}_ADOCDEPENDS";
  328. makevar_define($varname1);
  329. makevar_define($varname2);
  330. $depend_varname_hash->{$fn} = $varname2;
  331. makevar_list_add($varname1, $fn);
  332. makevar_list_add($varname2, "\$\{$varname1\}");
  333. foreach my $dep (sort keys %{$depends->{$fn}}) {
  334. if ($dep =~ m/-(opts|synopsis).adoc$/) {
  335. makevar_list_add($varname2, $dep);
  336. } else {
  337. makevar_list_add($varname1, $dep);
  338. }
  339. }
  340. }
  341. my $man_sources_hash = {};
  342. foreach my $sourcefile (keys %{$fileinfo->{outfile}->{manvolnum}}) {
  343. $man_sources_hash->{$sourcefile} = 1;
  344. my $ihash = $fileinfo->{include}->{manvolnum}->{$sourcefile};
  345. foreach my $include (keys %$ihash) {
  346. if ($include !~ m/-(opts|synopsis).adoc$/) {
  347. $man_sources_hash->{$include} = 1;
  348. }
  349. }
  350. }
  351. my $varname = "MANUAL_SOURCES";
  352. makevar_define($varname);
  353. foreach my $sourcefile (sort keys %$man_sources_hash) {
  354. makevar_list_add($varname, $sourcefile);
  355. }
  356. $varname = "CHAPTER_LIST";
  357. makevar_define($varname);
  358. my $filelist = $fileinfo->{outfile}->{default};
  359. foreach my $sourcefile (sort keys %$filelist) {
  360. my $target = $filelist->{$sourcefile};
  361. makevar_list_add($varname, $target);
  362. }
  363. $varname = "MANUAL_PAGES";
  364. makevar_define($varname);
  365. $filelist = $fileinfo->{outfile}->{manvolnum};
  366. foreach my $manpage (sort keys %$filelist) {
  367. my $target = $filelist->{$manpage};
  368. makevar_list_add($varname, $target);
  369. }
  370. $varname = "WIKI_IMPORTS";
  371. makevar_define($varname);
  372. $filelist = $fileinfo->{outfile}->{wiki};
  373. foreach my $sourcefile (sort keys %$filelist) {
  374. my $target = $filelist->{$sourcefile};
  375. makevar_list_add($varname, $target);
  376. }
  377. my $res = makevar_dump();
  378. my $make_targets = {};
  379. foreach my $e (@$start_env) {
  380. my $filelist = $fileinfo->{outfile}->{$e};
  381. foreach my $sourcefile (sort keys %$filelist) {
  382. my $varname = $depend_varname_hash->{$sourcefile};
  383. next if !defined($varname);
  384. my $target = $filelist->{$sourcefile};
  385. my $dep = "\$\{$varname\}";
  386. $make_targets->{$target} = $dep;
  387. if ($e eq 'manvolnum') {
  388. $make_targets->{"$target.html"} = $dep;
  389. $make_targets->{"$target-plain.html"} = $dep;
  390. }
  391. }
  392. }
  393. foreach my $target (sort keys%$make_targets) {
  394. my $dep = $make_targets->{$target};
  395. $res .= "$target: $dep\n\n";
  396. }
  397. print $outfh $res;
  398. close($outfh);
  399. rename($tmpfilename, $generate_depends) ||
  400. die "rename failed - $!";
  401. }
  402. print to_json($fileinfo, { pretty => 1, canonical => 1 } );