sitegen.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354
  1. import argparse
  2. from functools import lru_cache
  3. from importlib import import_module
  4. from jinja2 import Environment, FileSystemLoader, contextfilter
  5. from pathlib import Path
  6. from os import walk, makedirs, listdir, symlink, readlink
  7. from os.path import join, exists, splitext, split, islink, isdir
  8. from shutil import rmtree, copy2, copystat, ignore_patterns, Error
  9. from time import sleep
  10. from traceback import print_exc
  11. from urllib.parse import quote
  12. import os
  13. #TODO: load from config file (and watch it too)
  14. LANGUAGES = 'languages'
  15. PLUGINS = 'plugins'
  16. ROOT = 'root'
  17. STATIC = 'static'
  18. STATIC_URL = '/static'
  19. DEFAULT_LANG = 'en'
  20. OTHER_LANGS = set(['es'])
  21. WATCH_INTERVAL = 1 # in secs
  22. config = {
  23. 'LANGUAGES': LANGUAGES,
  24. 'PLUGINS': PLUGINS,
  25. 'ROOT': ROOT,
  26. 'STATIC': STATIC,
  27. 'DEFAULT_LANG': DEFAULT_LANG,
  28. 'OTHER_LANGS': OTHER_LANGS,
  29. 'WATCH_INTERVAL': WATCH_INTERVAL
  30. }
  31. # Utils
  32. ignore_underscores = ignore_patterns('_*')
  33. def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2,
  34. ignore_dangling_symlinks=False):
  35. """Recursively copy a directory tree.
  36. The destination directory must not already exist.
  37. If exception(s) occur, an Error is raised with a list of reasons.
  38. If the optional symlinks flag is true, symbolic links in the
  39. source tree result in symbolic links in the destination tree; if
  40. it is false, the contents of the files pointed to by symbolic
  41. links are copied. If the file pointed by the symlink doesn't
  42. exist, an exception will be added in the list of errors raised in
  43. an Error exception at the end of the copy process.
  44. You can set the optional ignore_dangling_symlinks flag to true if you
  45. want to silence this exception. Notice that this has no effect on
  46. platforms that don't support os.symlink.
  47. The optional ignore argument is a callable. If given, it
  48. is called with the `src` parameter, which is the directory
  49. being visited by copytree(), and `names` which is the list of
  50. `src` contents, as returned by os.listdir():
  51. callable(src, names) -> ignored_names
  52. Since copytree() is called recursively, the callable will be
  53. called once for each directory that is copied. It returns a
  54. list of names relative to the `src` directory that should
  55. not be copied.
  56. The optional copy_function argument is a callable that will be used
  57. to copy each file. It will be called with the source path and the
  58. destination path as arguments. By default, copy2() is used, but any
  59. function that supports the same signature (like copy()) can be used.
  60. """
  61. names = os.listdir(src)
  62. if ignore is not None:
  63. ignored_names = ignore(src, names)
  64. else:
  65. ignored_names = set()
  66. os.makedirs(dst, exist_ok=True)
  67. # `exist_ok=True`, very important for hot reloading
  68. errors = []
  69. for name in names:
  70. if name in ignored_names:
  71. continue
  72. srcname = os.path.join(src, name)
  73. dstname = os.path.join(dst, name)
  74. try:
  75. if os.path.islink(srcname):
  76. linkto = os.readlink(srcname)
  77. if symlinks:
  78. # We can't just leave it to `copy_function` because legacy
  79. # code with a custom `copy_function` may rely on copytree
  80. # doing the right thing.
  81. os.symlink(linkto, dstname)
  82. copystat(srcname, dstname, follow_symlinks=not symlinks)
  83. else:
  84. # ignore dangling symlink if the flag is on
  85. if not os.path.exists(linkto) and ignore_dangling_symlinks:
  86. continue
  87. # otherwise let the copy occurs. copy2 will raise an error
  88. if os.path.isdir(srcname):
  89. copytree(srcname, dstname, symlinks, ignore,
  90. copy_function)
  91. else:
  92. copy_function(srcname, dstname)
  93. elif os.path.isdir(srcname):
  94. copytree(srcname, dstname, symlinks, ignore, copy_function)
  95. else:
  96. # Will raise a SpecialFileError for unsupported file types
  97. copy_function(srcname, dstname)
  98. # catch the Error from the recursive copytree so that we can
  99. # continue with other files
  100. except Error as err:
  101. errors.extend(err.args[0])
  102. except OSError as why:
  103. errors.append((srcname, dstname, str(why)))
  104. try:
  105. copystat(src, dst)
  106. except OSError as why:
  107. # Copying file access times may fail on Windows
  108. if getattr(why, 'winerror', None) is None:
  109. errors.append((src, dst, str(why)))
  110. if errors:
  111. raise Error(errors)
  112. return dst
  113. def mtimes(target_dir):
  114. ''''get modification time of files in `target_dir`'''
  115. return { f: f.stat().st_mtime for f in Path(target_dir).rglob('*') }
  116. def save_dic(lang, dic):
  117. with open(join(LANGUAGES, lang + '.txt'), 'w') as f:
  118. for key, value in dic.items():
  119. f.write(f"{key}:{value}\n")
  120. @lru_cache(None)
  121. def get_lang_dic(lang):
  122. makedirs(LANGUAGES, exist_ok=True)
  123. lang_file = join(LANGUAGES, lang + '.txt')
  124. if not exists(lang_file):
  125. return {}
  126. else:
  127. dic = {}
  128. with open(lang_file) as f:
  129. for line in f.readlines():
  130. key, value = line.strip('\n').split(':')
  131. dic[key] = value
  132. return dic
  133. # Filters
  134. def add_lang_prefix(lang, path):
  135. if lang == DEFAULT_LANG:
  136. return path
  137. if lang not in OTHER_LANGS:
  138. print(f"Not registered language: `{lang}`")
  139. return f"/{lang}{path}"
  140. @contextfilter
  141. def lang(ctx, value):
  142. lang = ctx.environment.globals['lang']
  143. if lang == DEFAULT_LANG:
  144. return value
  145. else:
  146. dic = get_lang_dic(lang)
  147. if value in dic and dic[value]:
  148. return dic[value]
  149. else:
  150. dic[value] = ""
  151. save_dic(lang, dic)
  152. print(f"Not translated phrase: `{value}`")
  153. return value
  154. @contextfilter
  155. def lang_url(ctx, lang):
  156. name, _ = splitext(ctx.name)
  157. if name.startswith('./'):
  158. name = name[2:]
  159. if name == 'index':
  160. name = ''
  161. return add_lang_prefix(lang, f"/{name}")
  162. @contextfilter
  163. def cur_lang(ctx, path):
  164. lang = ctx.environment.globals['lang']
  165. return add_lang_prefix(lang, path)
  166. def static(value):
  167. parts = value.parts if isinstance(value, Path) else value.split('/')
  168. sub_path = '/'.join(map(lambda s: quote(s), parts))
  169. return f"{STATIC_URL}/{sub_path}"
  170. def svg(resource):
  171. with Path(STATIC, 'svg', f'{resource}.svg').open() as f:
  172. f.readline()
  173. f.readline()
  174. return f.read()
  175. def inline(file_name):
  176. return Path(file_name).read_text()
  177. # Compiler
  178. def compile(env, path, target):
  179. lang = env.globals['lang']
  180. for base, _, docs in walk(path):
  181. cur_base = Path(base)
  182. cur_base = Path(*cur_base.parts[1:])
  183. for doc in docs:
  184. if doc.startswith('_'):
  185. # Ignore this files
  186. continue
  187. src = join(base, doc)
  188. if args.with_index and doc != 'index.html':
  189. name, _ = splitext(doc)
  190. dst = join(cur_base, name, 'index.html')
  191. else:
  192. dst = join(cur_base, doc)
  193. if lang != DEFAULT_LANG:
  194. dst = join(lang, dst)
  195. dst = join(target, dst)
  196. dst_folder, _ = split(dst)
  197. makedirs(dst_folder, exist_ok=True)
  198. template = env.get_template(join(cur_base, doc))
  199. output = template.render()
  200. with open(dst, 'w') as f:
  201. f.write(output)
  202. def init_gen(args):
  203. # Load layout
  204. file_loader = FileSystemLoader(args.source)
  205. env = args.env = Environment(loader=file_loader)
  206. # Init gobals
  207. env.globals['lang'] = DEFAULT_LANG
  208. env.globals['list'] = list
  209. # Add filters
  210. env.filters['lang'] = lang
  211. env.filters['lang_url'] = lang_url
  212. env.filters['cur_lang'] = cur_lang
  213. env.filters['static'] = static
  214. env.filters['svg'] = svg
  215. env.filters['inline'] = inline
  216. # Load plugins
  217. for mod_path in Path(PLUGINS).glob('*.py'):
  218. mod_name = '.'.join(mod_path.with_suffix('').parts)
  219. print(f'* loading {mod_name}')
  220. import_module(mod_name).init_plugin(env, config)
  221. print(' done!')
  222. # Clean target
  223. if exists(args.target):
  224. rmtree(args.target)
  225. makedirs(args.target, exist_ok=True)
  226. def gen_layout(args):
  227. env = args.env
  228. path = Path(args.source)
  229. # Reset gobals
  230. env.globals['lang'] = DEFAULT_LANG
  231. # compile
  232. compile(env, path, args.target)
  233. for other_lang in OTHER_LANGS:
  234. env.globals['lang'] = other_lang
  235. compile(env, path, args.target)
  236. def gen_root(args):
  237. copytree(ROOT, args.target, ignore=ignore_underscores)
  238. def gen_static(args):
  239. copytree(STATIC, join(args.target, 'static'), ignore=ignore_underscores)
  240. def save_generate(generator, args, msg):
  241. print(f'* {msg}')
  242. try:
  243. generator(args)
  244. print(' done!')
  245. except:
  246. print_exc()
  247. return False
  248. return True
  249. # Runtime
  250. def run(args):
  251. print('initial compilation')
  252. init_gen(args)
  253. save_generate(gen_root, args, 'copying root files')
  254. save_generate(gen_static, args, 'copying static dir')
  255. save_generate(gen_layout, args, 'generating layout')
  256. if not args.watch:
  257. return
  258. print(f'watching {args.source}/*:{LANGUAGES}/*:{STATIC}/*:{ROOT}/*')
  259. while True:
  260. # take modification times
  261. source_mt = mtimes(args.source)
  262. source_mt.update(mtimes(LANGUAGES))
  263. static_mt = mtimes(STATIC)
  264. root_mt = mtimes(ROOT)
  265. sleep(WATCH_INTERVAL)
  266. # compare modification times and recompile if different
  267. new_source_mt = mtimes(args.source)
  268. new_source_mt.update(mtimes(LANGUAGES))
  269. if source_mt != new_source_mt:
  270. source_mt = new_source_mt
  271. save_generate(gen_layout, args, 'recompiling layout')
  272. new_root_mt = mtimes(ROOT)
  273. if root_mt != new_root_mt:
  274. root_mt = new_root_mt
  275. save_generate(gen_root, args, 'copying root files')
  276. new_static_mt = mtimes(STATIC)
  277. if static_mt != new_static_mt:
  278. static_mt = new_static_mt
  279. save_generate(gen_static, args, 'copying static files')
  280. save_generate(gen_layout, args, 'recompiling layout')
  281. if __name__ == '__main__':
  282. parser = argparse.ArgumentParser("Site generator")
  283. parser.add_argument('--watch', action='store_true', default=False)
  284. parser.add_argument('--with-index', action='store_true', default=False)
  285. parser.add_argument('--source', default='layout')
  286. parser.add_argument('--target', default='build')
  287. args = parser.parse_args()
  288. run(args)