[chirp_devel] [PATCH 03/11] Add pylint support to cpep8 scripts (#159)

8 Mar 2015

# HG changeset patch
# User Zachary T Welch zach@mandolincreekfarm.com
# Fake Node ID 56fbb6a0b29a1cbbc1c3a5feeb4ce501f4cedc6e
Add pylint support to cpep8 scripts (#159)
The patch extends the cpep8 scripts to include pylint.  Like pep8, it
imports the appropriate pylint modules and runs the checks in the same
python process.  It uses the pylintrc file that was updated in the last
patch to control its behavior.
By default, it only checks files from the manifest that do not appear in
its blacklist (tools/cpep8.lintful).  If --all is given, it scans all of
the files in the manifest.
It adds the --no-pep8 and --no-pylint options to limit manual testing to
one set of tests or the other.  The new --no-color option may be useful
when the output is not a terminal (e.g. Jenkins).
The cpep8.sh script overrides the pylint save-state directory, putting
the statistics into tools/cpep.pylint.d/.  As with the virtualenv files,
that directory can be overridden by setting TMPDIR.
Finally, it changes the verbose option to counting, so the pylint
reports for files with no errors are printed only when that option
appears twice.

diff --git a/.hgignore b/.hgignore
index 140091a..58a8f45 100644
--- a/.hgignore
+++ b/.hgignore
@@ -5,4 +5,5 @@
 dist
 build/bdist
 tools/cpep8.venv/
+tools/cpep8.pylint.d/
 tests/logs/
diff --git a/tools/cpep8.manifest b/tools/cpep8.lintful
similarity index 94%
copy from tools/cpep8.manifest
copy to tools/cpep8.lintful
index afeef1d..efb64b6 100644
--- a/tools/cpep8.manifest
+++ b/tools/cpep8.lintful
@@ -1,3 +1,6 @@
+# cpep8.lintful: The list of files that do not meet pylint standards.
+# DO NOT ADD NEW FILES!!  Instead, fix the code to be compliant.
+# Over time, this list should shrink and (eventually) be eliminated.
 ./chirp/__init__.py
 ./chirp/bandplan.py
 ./chirp/bandplan_au.py
diff --git a/tools/cpep8.py b/tools/cpep8.py
index 2bf55d0..655a17f 100755
--- a/tools/cpep8.py
+++ b/tools/cpep8.py
@@ -21,6 +21,8 @@ import os
 import sys
 import logging
 import argparse
+from pylint import lint, reporters
+from pylint.reporters import text
 import pep8
parser = argparse.ArgumentParser()
@@ -28,6 +30,12 @@ parser.add_argument("-a", "--all", action="store_true",
                     help="Check all files, ignoring blacklist")
 parser.add_argument("-d", "--dir", action="store", default=".",
                     help="Root directory of source tree")
+parser.add_argument("--no-color", action="store_true",
+                    help="Do not colorize output")
+parser.add_argument("--no-pep8", action="store_true",
+                    help="Do not run pep8 checks")
+parser.add_argument("--no-pylint", action="store_true",
+                    help="Do not run pylint checks")
 parser.add_argument("-s", "--stats", action="store_true",
                     help="Only show statistics")
 parser.add_argument("--strict", action="store_true",
@@ -36,7 +44,7 @@ parser.add_argument("-S", "--scan", action="store_true",
                     help="Scan for additional files")
 parser.add_argument("-u", "--update", action="store_true",
                     help="Update manifest/blacklist files")
-parser.add_argument("-v", "--verbose", action="store_true",
+parser.add_argument("-v", "--verbose", action="count", default=0,
                     help="Display list of checked files")
 parser.add_argument("files", metavar="file", nargs='*',
                     help="List of files to check (if none, check all)")
@@ -55,6 +63,8 @@ scriptdir = os.path.dirname(sys.argv[0])
 manifest_filename = os.path.join(scriptdir, "cpep8.manifest")
 blacklist_filename = os.path.join(scriptdir, "cpep8.blacklist")
 exceptions_filename = os.path.join(scriptdir, "cpep8.exceptions")
+lintful_filename = os.path.join(scriptdir, "cpep8.lintful")
+pylintrc_filename = os.path.join(scriptdir, "cpep8.pylintrc")
manifest = []
 if args.scan:
@@ -89,24 +99,93 @@ def get_exceptions(f):
         ignore = None
     return ignore
+
+class SmartReporter(reporters.BaseReporter):
+    """A pylint reporter that normally pipes output to /dev/null."""
+
+    def __init__(self, update):
+        reporters.BaseReporter.__init__(self)
+        self.update = update
+        self.stats = {}
+        self.msgs = {}
+        self.queue = []
+        self.reporter = args.no_color and \
+            text.TextReporter() or \
+            text.ColorizedTextReporter()
+
+    def on_set_current_module(self, module, path):
+        self.reporter.linter = self.linter
+        self.reporter.on_set_current_module(module, path)
+
+    def handle_message(self, msg):
+        msg_id = msg.msg_id
+        if msg_id not in self.stats:
+            self.stats[msg_id] = 0
+        self.stats[msg_id] += 1
+
+        if msg_id not in self.msgs:
+            self.msgs[msg_id] = msg.msg
+
+        self.queue.append(msg)
+
+    def add_message(self, dummy1, dummy2, dummy3):
+        pass
+
+    def _display(self, layout):
+        show_report = args.verbose > 1 or \
+                      ((not self.update or args.verbose) and
+                       len(self.queue) > 0)
+        if show_report:
+            for msg in self.queue:
+                self.reporter.handle_message(msg)
+            self.reporter.display_results(layout)
+        if args.stats:
+            for msgid in sorted(self.stats.keys()):
+                print "%d %s: %s" % \
+                      (self.stats[msgid], msgid, self.msgs[msgid])
+
+
 if args.update:
     print "Starting update of %d files" % len(manifest)
     bad = []
+    lintful = []
     for f in manifest:
         checker = pep8.StyleGuide(quiet=True, ignore=get_exceptions(f))
         results = checker.check_files([f])
         if results.total_errors:
             bad.append(f)
+
+        cmdline = [f, '--rcfile=%s' % pylintrc_filename]
+        reporter = SmartReporter(True)
+        runner = lint.Run(cmdline, reporter=reporter, exit=False)
+        if runner.linter.msg_status > 0:
+            results.total_errors += 1
+            lintful.append(f)
+
         print "%s: %s" % (results.total_errors and "FAIL" or "PASS", f)
-    with file(blacklist_filename, "w") as fh:
-        print >>fh, """\
-# cpep8.blacklist: The list of files that do not meet PEP8 standards.
+    do_not_edit = """\
 # DO NOT ADD NEW FILES!!  Instead, fix the code to be compliant.
 # Over time, this list should shrink and (eventually) be eliminated."""
-        print >>fh, "\n".join(sorted(bad))
+
+    print "Updating %s" % blacklist_filename
+    with file(blacklist_filename, "w") as fh:
+        print >>fh, """\
+# cpep8.blacklist: The list of files that do not meet PEP8 standards."""
+        print >>fh, do_not_edit
+        if len(bad) > 0:
+            print >>fh, "\n".join(sorted(bad))
+
+    print "Updating %s" % lintful_filename
+    with file(lintful_filename, "w") as fh:
+        print >>fh, """\
+# cpep8.lintful: The list of files that do not meet pylint standards."""
+        print >>fh, do_not_edit
+        if len(lintful) > 0:
+            print >>fh, "\n".join(sorted(lintful))
if args.scan:
+        print "Updating %s" % manifest_filename
         with file(manifest_filename, "w") as fh:
             print >>fh, "\n".join(sorted(manifest))
     sys.exit(0)
@@ -116,17 +195,22 @@ if args.files:
# read the blacklisted source files
 blacklist = file_to_lines(blacklist_filename)
+lintful = file_to_lines(lintful_filename)
check_list = []
+lint_list = []
 for f in manifest:
-    if args.all or f not in blacklist:
+    if not args.no_pep8 and (args.all or f not in blacklist):
         check_list.append(f)
+    if not args.no_pylint and (args.all or f not in lintful):
+        lint_list.append(f)
 check_list = sorted(check_list)
+lint_list = sorted(lint_list)
total_errors = 0
 for f in check_list:
     if args.verbose:
-        print "Checking %s" % f
+        print "pep8: checking %s" % f
checker = pep8.Checker(f, quiet=args.stats, ignore=get_exceptions(f))
     results = checker.check_all()
@@ -134,4 +218,13 @@ for f in check_list:
         checker.report.print_statistics()
     total_errors += results
+for f in lint_list:
+    if args.verbose:
+        print "pylint: checking %s" % f
+    cmdline = [f, '--rcfile=%s' % pylintrc_filename]
+    reporter = SmartReporter(False)
+    runner = lint.Run(cmdline, reporter=reporter, exit=False)
+    if runner.linter.msg_status > 0:
+        total_errors += 1
+
 sys.exit(total_errors and 1 or 0)
diff --git a/tools/cpep8.sh b/tools/cpep8.sh
index 89fd9ac..938c1c6 100755
--- a/tools/cpep8.sh
+++ b/tools/cpep8.sh
@@ -2,9 +2,11 @@
 # Runs cpep.py with the proper verion of the pep8 library.
PEP8_VERSION="1.6.2"
+PYLINT_VERSION="1.4.1"
TOOLS_DIR="$(dirname $0)"
 VENV="${TMPDIR:-${TOOLS_DIR}}/cpep8.venv"
+export PYLINTHOME="${TMPDIR:-${TOOLS_DIR}}/cpep8.pylint.d"
virtualenv="$(which virtualenv)"
 if [ ! -x "$virtualenv" ]; then
@@ -13,9 +15,15 @@ if [ ! -x "$virtualenv" ]; then
 fi
 if [ ! -d "$VENV" ]; then
     virtualenv "$VENV"
+    source ${VENV}/bin/activate
+    mkdir -p ${VENV}/logs
+    pip install pep8==${PEP8_VERSION} >${VENV}/logs/pep8.log 2>&1
+    pip install pylint==${PYLINT_VERSION} >${VENV}/logs/pylint.log 2>&1
+    pip install pyserial >${VENV}/logs/pyserial.log 2>&1
+    pip install pygtk >${VENV}/logs/pygtk.log 2>&1
+else
+    source ${VENV}/bin/activate
 fi
-source ${VENV}/bin/activate
-pip install pep8==${PEP8_VERSION} >${VENV}/pep8.log 2>&1
 ${TOOLS_DIR}/cpep8.py "$@"
 deactivate

    