pyccoon v0.1.7 documentation: pyccoon.py

#

#!/usr/bin/env python
# -*- coding: utf-8 -*-

#

Pyccoon

"Pyccoon" is a side-to-side documentation generator.

import optparse
import os
import shutil
import pystache
import re
import sys
import json
from io import open
from datetime import datetime
from collections import defaultdict

#

This module contains all of our static resources.

from . import resources, __version__, __author__
from .languages import get_language, Language

from .utils import shift, ensure_directory, SourceFile

#

Main documentation generation class

class Pyccoon(object):

    add_lineno = True

#

The start of each Pygments highlight block.

    highlight_start = "<div class=\"highlight\"><pre>"

#

The end of each Pygments highlight block.

    highlight_end = "</pre></div>"

    config = defaultdict(lambda: [], {
        "skip_files": [".+\\.pyc", "__pycache__", "\\.travis.yml", "\\.git", "\\.DS_Store"]
    })

    config_file = '.pyccoon'
    watch = False
    verbosity = -1

    outdir = sourcedir = None

#

Add cross-references before having the text processed by markdown. It's possible to reference another file, like this : [utils.py](utils.py.html) which renders utils.py. You can also reference a specific section of another file, like this: [utils.py#ensure-directory](utils.py.html#ensure-directory) which renders as utils.py#ensure-directory. Sections have to be manually declared; they are written on a single line, prefixed by #s: ### like this

    def preprocess(self, comment, source):

#

Return URL-friendly section name representation

        def slugify(name):

            return "-".join(name.lower().strip().split(" "))

#

        def replace_crossref(match):
            name = match.group(1)
            if name:
                name = name.rstrip("|")
            path = match.group(2)

            if not name and not path:
                return

#

Check if the match contains an anchor

            anchor = None
            if '#' in path:
                path, anchor = path.split('#')

            if not name:
                name = os.path.basename(path)
                if anchor:
                    name = name + '#' + anchor

            anchor = '#' + anchor if anchor else ''

            if not path.startswith('.'):

#

Absolute reference

                path = os.path.relpath(
                    self.destination(path),
                    os.path.split(self.sources[os.path.relpath(source,
                                                               self.sourcedir)].destination)[0]
                )

#

            else:

#

Relative reference

                path = os.path.relpath(
                    self.destination(os.path.join(
                        os.path.split(os.path.relpath(source, self.sourcedir))[0], path)
                    ),
                    os.path.split(self.sources[os.path.relpath(source,
                                                               self.sourcedir)].destination)[0]
                )

#

            return "[{0:s}]({1:s}{2:s})".format(name, path, anchor)

#

        def replace_section_name(match):
            return (
                    '\n{lvl} <a id="{id}" class="header-anchor" href="#{id}">{name}</a>'
            ).format(**{
                "lvl":  match.group(2),
                "id":   slugify(match.group(3)),
                "name": match.group(3)
            })

#

def replace_texblocks(match): print match.groups() return ( '\n{begin}\n{code}\n{end}\n' ).format(**{ "begin": r"\begin{{{}}}".format(match.group(2)), "end": r"\end{{{}}}".format(match.group(2)), "code": match.group(3) })

        comment = re.compile(r'^\s*(#\s)?\s*(#+)([^#\n]+)\s*$', re.M)\
            .sub(replace_section_name, comment)
        comment = re.sub(r'\[\[([^\|\n]+\|)?(.+?)\]\]', replace_crossref, comment)

#

comment = re.compile(r'\stex(`([\w]+))?([\s\S]+)\s$', re.M) .sub(replace_texblocks, comment)

        return comment

#

HTML Code generation

#

Once all of the code is finished highlighting, we can generate the HTML file and write out the documentation. Pass the completed sections into the template found in resources/pyccoon.html.

Pystache will attempt to recursively render context variables, so we must replace any occurences of {{, which is valid in some languages, with a "unique enough" identifier before rendering, and then post-process the rendered template and change the identifier back to {{.

    def generate_html(self, source, sections):

        dest = self.destination(source)
        title = os.path.relpath(source, self.sourcedir)
        page_title = self.project_name + ": " + os.path.relpath(source, self.sourcedir).lstrip('./')
        csspath = os.path.relpath(os.path.join(self.outdir, resources.css_filename),
                                  os.path.split(dest)[0])

        breadcrumbs, filename = self.generate_breadcrumbs(dest, title)
        children = self.generate_navigation(source)
        contents = self.generate_contents(sections)

        for section in sections:
            section['line_count'] = (section['code_text'].rstrip('\n') + '\n').count('\n')
            section['linenos'] = '\n'.join(str(section['line'] + i)
                                           for i in range(section['line_count']))

        rendered = self.page_template({
            "title":            page_title,
            "breadcrumbs":      breadcrumbs,
            "filename":         filename,
            "children":         children,
            "stylesheet":       csspath,
            "sections":         sections,
            "source":           source,
            "contents":         contents,
            "contents?":        bool(contents),
            "destination":      dest,
            "generation_time":  datetime.now().strftime('%Y-%m-%d %H:%M'),
            "root_path":        os.path.relpath(".", os.path.split(source)[0]),
            "project_name":     self.project_name,
            "mathjax?":          self.config['mathjax'],
            "docs_only?": not any(section['code_text'] for section in sections)
        })

        return rendered.replace("{{", "{{")

#

Generating breadcrumbs

Based on the source file path, generate linked breadcrumbs of the documentation.

    def generate_breadcrumbs(self, dest, title):

        breadcrumbs = []
        crumbpath = None

        dest_chunks = os.path.relpath(dest, self.outdir).split("/")
        source_chunks = title.split("/")
        dest_chunks.reverse()
        source_chunks.reverse()

        for i, crumb in enumerate(dest_chunks):

            crumbpath = os.path.join(crumbpath, "..") if crumbpath else crumb

            breadcrumbs.insert(0, {
                "title": source_chunks[i],
                "path": crumbpath if crumbpath.endswith('.html')
                else os.path.join(crumbpath, 'index.html')
            })
        breadcrumbs.insert(0, {
            "title": ".",
            "path": os.path.join(crumbpath, "../index.html")
        })

        return breadcrumbs[:-1], source_chunks[0]

#

Generating navigation

For index.html files, generate a menu of folder contents.

TODO: remove language dependency

    def generate_navigation(self, source):

        index_names = [r'__init__\..+', r'index\..+']
        basename = os.path.basename(source)
        if not any([re.match(regex, basename) for regex in index_names]):
            return []

        children = []
        folder = os.path.split(os.path.join(self.sourcedir, source))[0]
        relfolder = os.path.relpath(folder, self.sourcedir)
        outfolder = os.path.join(self.outdir, relfolder) if relfolder != "." else self.outdir
        for filename in os.listdir(folder):
            if not any([regex.search(filename) for regex in self.config['skip_files']]):
                isdir = False
                filepath = None

                if os.path.isdir(os.path.join(folder, filename)):
                    isdir = True
                    filepath = os.path.join(filename, "index.html")
                else:

                    if filename in index_names:
                        filepath = "index.html"
                    else:
                        in_sources = self.sources.get(
                            os.path.join(relfolder, filename)
                            if relfolder != "." else filename
                        )

                        if in_sources:
                            filepath = in_sources.destination[len(outfolder)+1:]

                if filepath:
                    children.append({
                        "title": filename,
                        "path": filepath,
                        "isdir": isdir
                    })

        return sorted(children, key=lambda x: not x['isdir'])

#

Generating page contents

Gather the names of the documentation sections for "jump-to"-like navigation on the page.

    def generate_contents(self, sections):

        contents = []

        for section in sections:
            section["code_html"] = section["code_html"].replace("{{", "{{")

            for match in re.finditer(r'<h(\d)>(.+href=\"#(.+)\".+)</h(\d)>',
                                     section["docs_html"], re.M):

                contents.append({
                    "url": "#{0}".format(match.group(3)),
                    "basename": re.sub(r'<[^<]+?>', '', match.group(2)),
                    "level": match.group(1)
                })
        return contents

#

Utilities

#

Compute the destination HTML path for an input source file path. If the source is lib/example.py, the HTML will be at docs/lib/example.html

    def destination(self, source, language=None, process=True):

        dirname, filename = os.path.split(source)
        if process:
            language = language or self.get_language(source)

        name = language.transform_filename(filename) if language else filename
        return os.path.normpath(os.path.join(self.outdir, os.path.join(dirname, name)))

#

Determine language of the file

    def get_language(self, source):

        language = None

        try:
            with open(os.path.join(self.sourcedir, source), "rb") as sourcefile:
                code = sourcefile.read().decode('utf8')

            language = get_language(source, code)
        except Exception:
            pass

        return language

#

Hook spot for the console script.

def main():

    parser = optparse.OptionParser(version='Pyccoon {0}'.format(__version__))
    parser.add_option('-s', '--source', action='store', type='string',
                      dest='sourcedir', default='.',
                      help='Source files directory (default: `%default`)')

    parser.add_option('-d', '--destination', action='store', type='string',
                      dest='outdir', default='docs',
                      help='Output directory (default: `%default`)')

    parser.add_option('-w', '--watch', action='store_true',
                      help='Watch original files and regenerate documentation on changes')

    parser.add_option('-c', '--config', action='store', dest='config_file',
                      default=os.path.join(os.getcwd(), '.pyccoon'), type='string',
                      help='Config file to use (default: `%default`)')

    parser.add_option('-v', '--verbosity', action='store', dest='verbosity',
                      default=-1, type='int',
                      help='Terminal output verbosity (0 to 1; default: %default)')

    opts, _ = parser.parse_args()
    opts = defaultdict(lambda: None, vars(opts))

    Pyccoon(opts)

#

Run the script.

if __name__ == "__main__":
    main()

Main documentation generation class

Pyccoon initialization

Source files processing

Generating documentation

Highlighting the source code

Preprocessing the comments

HTML Code generation

Generating breadcrumbs

Generating navigation

Generating page contents

Utilities