From 2ce8921fe69ed58871f8e33e3899ad80dcc28c0e Mon Sep 17 00:00:00 2001 From: Martin Basti Date: Fri, 15 Jan 2016 16:58:38 +0100 Subject: [PATCH] make lint: use config file and plugin for pylint Our custom implementation of pylint checker is often broken by incompatible change on pylint side. Using supported solutions (config file, pylint plugins) should avoid this issue. The plugin adds missing (dynamic) member to classes in abstract syntax tree generated for pylint, instead of just ignoring missing members and all sub-members. This should improve pylint detection of typos and missing members in api. env and test config. make-lint python script has been removed, to run pylint execute 'make lint' https://fedorahosted.org/freeipa/ticket/5615 Reviewed-By: David Kupka Reviewed-By: Jan Cholasta --- Makefile | 14 +- ipalib/plugins/vault.py | 4 + ipatests/test_ipapython/test_ipautil.py | 2 + make-lint | 351 ------------------------ pylint_plugins.py | 210 ++++++++++++++ pylintrc | 85 ++++++ 6 files changed, 312 insertions(+), 354 deletions(-) delete mode 100755 make-lint create mode 100644 pylint_plugins.py create mode 100644 pylintrc diff --git a/Makefile b/Makefile index d2857ae47..8c1a5dea7 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,9 @@ LIBDIR ?= /usr/lib DEVELOPER_MODE ?= 0 ifneq ($(DEVELOPER_MODE),0) -LINT_OPTIONS=--no-fail +LINT_IGNORE_FAIL=true +else +LINT_IGNORE_FAIL=false endif PYTHON ?= $(shell rpm -E %__python || echo /usr/bin/python2) @@ -127,8 +129,14 @@ client-dirs: fi lint: bootstrap-autogen - ./make-lint $(LINT_OPTIONS) - $(MAKE) -C install/po validate-src-strings + # find all python modules and executable python files outside modules for pylint check + FILES=`find . \ + -type d -exec test -e '{}/__init__.py' \; -print -prune -o \ + -name \*.py -print -o \ + -type f \! -path '*/.*' \! -name '*~' -exec grep -qsm1 '^#!.*\bpython' '{}' \; -print`; \ + echo "Pylint is running, please wait ..."; \ + PYTHONPATH=. pylint --rcfile=pylintrc $(PYLINTFLAGS) $$FILES || $(LINT_IGNORE_FAIL) + $(MAKE) -C install/po validate-src-strings || $(LINT_IGNORE_FAIL) test: diff --git a/ipalib/plugins/vault.py b/ipalib/plugins/vault.py index 4d8419e75..f495aa1cb 100644 --- a/ipalib/plugins/vault.py +++ b/ipalib/plugins/vault.py @@ -1651,7 +1651,9 @@ class vault_archive(PKQuery, Local): session_key = slot.key_gen(mechanism, None, key_length) # wrap session key with transport certificate + # pylint: disable=no-member public_key = nss_transport_cert.subject_public_key_info.public_key + # pylint: enable=no-member wrapped_session_key = nss.pub_wrap_sym_key(mechanism, public_key, session_key) @@ -1855,7 +1857,9 @@ class vault_retrieve(PKQuery, Local): session_key = slot.key_gen(mechanism, None, key_length) # wrap session key with transport certificate + # pylint: disable=no-member public_key = nss_transport_cert.subject_public_key_info.public_key + # pylint: enable=no-member wrapped_session_key = nss.pub_wrap_sym_key(mechanism, public_key, session_key) diff --git a/ipatests/test_ipapython/test_ipautil.py b/ipatests/test_ipapython/test_ipautil.py index 1197d5ce5..8a437f83e 100644 --- a/ipatests/test_ipapython/test_ipautil.py +++ b/ipatests/test_ipapython/test_ipautil.py @@ -384,6 +384,8 @@ class TestTimeParser(object): nose.tools.assert_equal(800000, time.microsecond) def test_time_zones(self): + # pylint: disable=no-member + timestr = "20051213141205Z" time = ipautil.parse_generalized_time(timestr) diff --git a/make-lint b/make-lint deleted file mode 100755 index cbc7d285a..000000000 --- a/make-lint +++ /dev/null @@ -1,351 +0,0 @@ -#!/usr/bin/python2 -# -# Authors: -# Jakub Hrozek -# Jan Cholasta -# -# Copyright (C) 2011 Red Hat -# see file 'COPYING' for use and warranty information -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from __future__ import print_function - -import os -import sys -from optparse import OptionParser -from fnmatch import fnmatch, fnmatchcase -import subprocess - -try: - from pylint import checkers - from pylint.lint import PyLinter - from pylint.checkers.typecheck import TypeChecker - from pylint.checkers.utils import safe_infer - from astroid import Class, Instance, Module, InferenceError, Function - from pylint.reporters.text import TextReporter -except ImportError: - print("To use {0}, please install pylint.".format(sys.argv[0]), file=sys.stderr) - sys.exit(32) - -# File names to ignore when searching for python source files -IGNORE_FILES = ('.*', '*~', '*.in', '*.pyc', '*.pyo') -IGNORE_PATHS = ( - 'build', 'rpmbuild', 'dist', 'install/po/test_i18n.py', 'lite-server.py') - -class IPATypeChecker(TypeChecker): - NAMESPACE_ATTRS = ['Command', 'Object', 'Method', 'Backend', 'Updater', - 'Advice'] - LOGGING_ATTRS = ['log', 'debug', 'info', 'warning', 'error', 'exception', - 'critical'] - - # 'class or module': ['generated', 'properties'] - ignore = { - # Python standard library & 3rd party classes - 'socket._socketobject': ['sendall'], - # should be 'subprocess.Popen' - '.Popen': ['stdin', 'stdout', 'stderr', 'pid', 'returncode', 'poll', - 'wait', 'communicate'], - 'urlparse.ResultMixin': ['scheme', 'netloc', 'path', 'query', - 'fragment', 'username', 'password', 'hostname', 'port'], - 'urlparse.ParseResult': ['params'], - 'pytest': ['fixture', 'raises', 'skip', 'yield_fixture', 'mark', 'fail'], - 'unittest.case': ['assertEqual', 'assertRaises'], - 'nose.tools': ['assert_equal', 'assert_raises'], - 'datetime.tzinfo': ['houroffset', 'minoffset', 'utcoffset', 'dst'], - 'nss.nss.subject_public_key_info': ['public_key'], - - # IPA classes - 'ipalib.base.NameSpace': ['add', 'mod', 'del', 'show', 'find'], - 'ipalib.cli.Collector': ['__options'], - 'ipalib.config.Env': ['*'], - 'ipalib.parameters.Param': ['cli_name', 'cli_short_name', 'label', - 'default', 'doc', 'required', 'multivalue', 'primary_key', - 'normalizer', 'default_from', 'autofill', 'query', 'attribute', - 'include', 'exclude', 'flags', 'hint', 'alwaysask', 'sortorder', - 'csv', 'option_group'], - 'ipalib.parameters.Bool': ['truths', 'falsehoods'], - 'ipalib.parameters.Data': ['minlength', 'maxlength', 'length', - 'pattern', 'pattern_errmsg'], - 'ipalib.parameters.Str': ['noextrawhitespace'], - 'ipalib.parameters.Password': ['confirm'], - 'ipalib.parameters.File': ['stdin_if_missing'], - 'ipalib.plugins.dns.DNSRecord': ['validatedns', 'normalizedns'], - 'ipalib.parameters.Enum': ['values'], - 'ipalib.parameters.Number': ['minvalue', 'maxvalue'], - 'ipalib.parameters.Decimal': ['precision', 'exponential', - 'numberclass'], - 'ipalib.parameters.DNSNameParam': ['only_absolute', 'only_relative'], - 'ipalib.plugable.API': NAMESPACE_ATTRS + LOGGING_ATTRS, - 'ipalib.plugable.Plugin': ['api', 'env'] + NAMESPACE_ATTRS + - LOGGING_ATTRS, - 'ipalib.session.AuthManager': LOGGING_ATTRS, - 'ipalib.session.SessionAuthManager': LOGGING_ATTRS, - 'ipalib.session.SessionManager': LOGGING_ATTRS, - 'ipaserver.install.ldapupdate.LDAPUpdate': LOGGING_ATTRS, - 'ipaserver.rpcserver.KerberosSession': ['api'] + LOGGING_ATTRS, - 'ipatests.test_integration.base.IntegrationTest': [ - 'domain', 'master', 'replicas', 'clients', 'ad_domains'] - } - - def _related_classes(self, klass): - yield klass - for base in klass.ancestors(): - yield base - - def _class_full_name(self, klass): - return klass.root().name + '.' + klass.name - - def _find_ignored_attrs(self, owner): - attrs = [] - for klass in self._related_classes(owner): - name = self._class_full_name(klass) - if name in self.ignore: - attrs += self.ignore[name] - return attrs - - def visit_getattr(self, node): - try: - inferred = list(node.expr.infer()) - except InferenceError: - inferred = [] - - for owner in inferred: - if isinstance(owner, Module): - if node.attrname in self.ignore.get(owner.name, ()): - return - - elif isinstance(owner, Class) or type(owner) is Instance: - ignored = self._find_ignored_attrs(owner) - for pattern in ignored: - if fnmatchcase(node.attrname, pattern): - return - - super(IPATypeChecker, self).visit_getattr(node) - - def visit_callfunc(self, node): - called = safe_infer(node.func) - if isinstance(called, Function): - if called.name in self.ignore.get(called.root().name, []): - return - - super(IPATypeChecker, self).visit_callfunc(node) - -class IPALinter(PyLinter): - ignore = (TypeChecker,) - - def __init__(self): - super(IPALinter, self).__init__() - - self.missing = set() - - def register_checker(self, checker): - if type(checker) in self.ignore: - return - super(IPALinter, self).register_checker(checker) - - def add_message(self, msg_id, line=None, node=None, args=None, confidence=None): - if line is None and node is not None: - line = node.fromlineno - - # Record missing packages - if msg_id == 'F0401' and self.is_message_enabled(msg_id, line): - self.missing.add(args) - - super(IPALinter, self).add_message(msg_id, line, node, args) - -def find_files(path, basepath): - entries = os.listdir(path) - - # If this directory is a python package, look no further - if '__init__.py' in entries: - return [path] - - result = [] - for filename in entries: - filepath = os.path.join(path, filename) - - for pattern in IGNORE_FILES: - if fnmatch(filename, pattern): - filename = None - break - if filename is None: - continue - - for pattern in IGNORE_PATHS: - patpath = os.path.join(basepath, pattern).replace(os.sep, '/') - if filepath == patpath: - filename = None - break - if filename is None: - continue - - if os.path.islink(filepath): - continue - - # Recurse into subdirectories - if os.path.isdir(filepath): - result += find_files(filepath, basepath) - continue - - # Add all *.py files - if filename.endswith('.py'): - result.append(filepath) - continue - - # Add any other files beginning with a shebang and having - # the word "python" on the first line - file = open(filepath, 'r') - line = file.readline(128) - file.close() - - if line[:2] == '#!' and line.find('python') >= 0: - result.append(filepath) - - return result - -def main(): - optparser = OptionParser() - optparser.add_option('--no-fail', help='report success even if errors were found', - dest='fail', default=True, action='store_false') - optparser.add_option('--enable-noerror', help='enable warnings and other non-error messages', - dest='errors_only', default=True, action='store_false') - optparser.add_option('--no-py3k', help='Do not check for Python 3 porting issues', - dest='py3k', default=True, action='store_false') - optparser.add_option('--no-lint', help='Skip the main lint check', - dest='do_lint', default=True, action='store_false') - - options, args = optparser.parse_args() - cwd = os.getcwd() - - if len(args) == 0: - files = find_files(cwd, cwd) - else: - files = args - - for filename in files: - dirname = os.path.dirname(filename) - if dirname not in sys.path: - sys.path.insert(0, dirname) - - linter = IPALinter() - checkers.initialize(linter) - linter.register_checker(IPATypeChecker(linter)) - - if options.errors_only: - linter.set_option('disable', 'I') - - linter.set_option('enable', 'F') - linter.set_option('disable', 'import-error') - - linter.set_option('enable', 'E') - - linter.set_option('enable', 'W') - linter.set_option('disable', 'abstract-method') - linter.set_option('disable', 'anomalous-backslash-in-string') - linter.set_option('disable', 'arguments-differ') - linter.set_option('disable', 'attribute-defined-outside-init') - linter.set_option('disable', 'bad-builtin') - linter.set_option('disable', 'bad-indentation') - linter.set_option('disable', 'bare-except') - linter.set_option('disable', 'broad-except') - linter.set_option('disable', 'dangerous-default-value') - linter.set_option('disable', 'eval-used') - linter.set_option('disable', 'exec-used') - linter.set_option('disable', 'fixme') - linter.set_option('disable', 'global-statement') - linter.set_option('disable', 'global-variable-not-assigned') - linter.set_option('disable', 'global-variable-undefined') - linter.set_option('disable', 'no-init') - linter.set_option('disable', 'pointless-except') - linter.set_option('disable', 'pointless-statement') - linter.set_option('disable', 'pointless-string-statement') - linter.set_option('disable', 'protected-access') - linter.set_option('disable', 'redefine-in-handler') - linter.set_option('disable', 'redefined-builtin') - linter.set_option('disable', 'redefined-outer-name') - linter.set_option('disable', 'reimported') - linter.set_option('disable', 'relative-import') - linter.set_option('disable', 'super-init-not-called') - linter.set_option('disable', 'undefined-loop-variable') - linter.set_option('disable', 'unnecessary-lambda') - linter.set_option('disable', 'unnecessary-semicolon') - linter.set_option('disable', 'unused-argument') - linter.set_option('disable', 'unused-variable') - linter.set_option('disable', 'useless-else-on-loop') - - linter.set_option('enable', 'C') - linter.set_option('disable', 'bad-classmethod-argument') - linter.set_option('disable', 'bad-continuation') - linter.set_option('disable', 'bad-mcs-classmethod-argument') - linter.set_option('disable', 'bad-mcs-method-argument') - linter.set_option('disable', 'bad-whitespace') - linter.set_option('disable', 'blacklisted-name') - linter.set_option('disable', 'invalid-name') - linter.set_option('disable', 'line-too-long') - linter.set_option('disable', 'missing-docstring') - linter.set_option('disable', 'multiple-imports') - linter.set_option('disable', 'multiple-statements') - linter.set_option('disable', 'old-style-class') - linter.set_option('disable', 'superfluous-parens') - linter.set_option('disable', 'too-many-lines') - linter.set_option('disable', 'unidiomatic-typecheck') - - linter.set_option('disable', 'R') - - linter.set_reporter(TextReporter()) - linter.set_option('msg-template', - '{path}:{line}: [{msg_id}({symbol}), {obj}] {msg})') - linter.set_option('reports', False) - linter.set_option('persistent', False) - linter.set_option('disable', 'python3') - - if options.do_lint: - linter.check(files) - - if linter.msg_status != 0: - print(""" -=============================================================================== -Errors were found during the static code check. -""", file=sys.stderr) - - if len(linter.missing) > 0: - print("There are some missing imports:", file=sys.stderr) - for mod in sorted(linter.missing): - print(" " + mod, file=sys.stderr) - print(""" -Please make sure all of the required and optional (python-gssapi, python-rhsm) -python packages are installed. -""", file=sys.stderr) - - print("""\ -If you are certain that any of the reported errors are false positives, please -mark them in the source code according to the pylint documentation. -=============================================================================== -""", file=sys.stderr) - - if options.fail and linter.msg_status != 0: - return linter.msg_status - - if options.py3k: - args = ['pylint', '--py3k', '-d', 'no-absolute-import', '--reports=n'] - args.extend(files) - returncode = subprocess.call(args) - if options.fail and returncode != 0: - return returncode - - return 0 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/pylint_plugins.py b/pylint_plugins.py new file mode 100644 index 000000000..f64c426fd --- /dev/null +++ b/pylint_plugins.py @@ -0,0 +1,210 @@ +# +# Copyright (C) 2015 FreeIPA Contributors see COPYING for license +# + +from __future__ import print_function + +import copy +import sys + +from astroid import MANAGER +from astroid import scoped_nodes + + +def register(linter): + pass + + +def _warning_already_exists(cls, member): + print( + "WARNING: member '{member}' in '{cls}' already exists".format( + cls="{}.{}".format(cls.root().name, cls.name), member=member), + file=sys.stderr + ) + + +def fake_class(name_or_class_obj, members=()): + if isinstance(name_or_class_obj, scoped_nodes.Class): + cl = name_or_class_obj + else: + cl = scoped_nodes.Class(name_or_class_obj, None) + + for m in members: + if isinstance(m, str): + if m in cl.locals: + _warning_already_exists(cl, m) + else: + cl.locals[m] = [scoped_nodes.Class(m, None)] + elif isinstance(m, dict): + for key, val in m.items(): + assert isinstance(key, str), "key must be string" + if key in cl.locals: + _warning_already_exists(cl, key) + fake_class(cl.locals[key], val) + else: + cl.locals[key] = [fake_class(key, val)] + else: + # here can be used any astroid type + if m.name in cl.locals: + _warning_already_exists(cl, m.name) + else: + cl.locals[m.name] = [copy.copy(m)] + return cl + + +fake_backend = {'Backend': [ + {'wsgi_dispatch': ['mount']}, +]} + +NAMESPACE_ATTRS = ['Command', 'Object', 'Method', fake_backend, 'Updater', + 'Advice'] +fake_api_env = {'env': [ + 'host', + 'realm', + 'session_auth_duration', + 'session_duration_type', +]} + +# this is due ipaserver.rpcserver.KerberosSession where api is undefined +fake_api = {'api': [fake_api_env] + NAMESPACE_ATTRS} + +_LOGGING_ATTRS = ['debug', 'info', 'warning', 'error', 'exception', + 'critical'] +LOGGING_ATTRS = [ + {'log': _LOGGING_ATTRS}, +] + _LOGGING_ATTRS + +# 'class': ['generated', 'properties'] +ipa_class_members = { + # Python standard library & 3rd party classes + 'socket._socketobject': ['sendall'], + + # IPA classes + 'ipalib.base.NameSpace': [ + 'add', + 'mod', + 'del', + 'show', + 'find' + ], + 'ipalib.cli.Collector': ['__options'], + 'ipalib.config.Env': [ + {'__d': ['get']}, + {'__done': ['add']}, + 'xmlrpc_uri', + 'validate_api', + 'startup_traceback', + 'verbose' + ] + LOGGING_ATTRS, + 'ipalib.parameters.Param': [ + 'cli_name', + 'cli_short_name', + 'label', + 'default', + 'doc', + 'required', + 'multivalue', + 'primary_key', + 'normalizer', + 'default_from', + 'autofill', + 'query', + 'attribute', + 'include', + 'exclude', + 'flags', + 'hint', + 'alwaysask', + 'sortorder', + 'csv', + 'option_group', + ], + 'ipalib.parameters.Bool': [ + 'truths', + 'falsehoods'], + 'ipalib.parameters.Data': [ + 'minlength', + 'maxlength', + 'length', + 'pattern', + 'pattern_errmsg', + ], + 'ipalib.parameters.Str': ['noextrawhitespace'], + 'ipalib.parameters.Password': ['confirm'], + 'ipalib.parameters.File': ['stdin_if_missing'], + 'ipalib.plugins.dns.DNSRecord': [ + 'validatedns', + 'normalizedns', + ], + 'ipalib.parameters.Enum': ['values'], + 'ipalib.parameters.Number': [ + 'minvalue', + 'maxvalue', + ], + 'ipalib.parameters.Decimal': [ + 'precision', + 'exponential', + 'numberclass', + ], + 'ipalib.parameters.DNSNameParam': [ + 'only_absolute', + 'only_relative', + ], + 'ipalib.plugable.API': [ + fake_api_env, + ] + NAMESPACE_ATTRS + LOGGING_ATTRS, + 'ipalib.plugable.Plugin': [ + 'Object', + 'Method', + 'Updater', + 'Advice', + ] + LOGGING_ATTRS, + 'ipalib.session.AuthManager': LOGGING_ATTRS, + 'ipalib.session.SessionAuthManager': LOGGING_ATTRS, + 'ipalib.session.SessionManager': LOGGING_ATTRS, + 'ipaserver.install.ldapupdate.LDAPUpdate': LOGGING_ATTRS, + 'ipaserver.rpcserver.KerberosSession': [ + fake_api, + ] + LOGGING_ATTRS, + 'ipatests.test_integration.base.IntegrationTest': [ + 'domain', + {'master': [ + {'config': [ + {'dirman_password': dir(str)}, + {'admin_password': dir(str)}, + {'admin_name': dir(str)}, + {'dns_forwarder': dir(str)}, + {'test_dir': dir(str)}, + {'ad_admin_name': dir(str)}, + {'ad_admin_password': dir(str)}, + {'domain_level': dir(str)}, + ]}, + {'domain': [ + {'realm': dir(str)}, + {'name': dir(str)}, + ]}, + 'hostname', + 'ip', + 'collect_log', + {'run_command': [ + {'stdout_text': dir(str)}, + 'stderr_text', + 'returncode', + ]}, + {'transport': ['put_file']}, + 'put_file_contents', + 'get_file_contents', + ]}, + 'replicas', + 'clients', + 'ad_domains', + ] +} + + +def fix_ipa_classes(cls): + class_name_with_module = "{}.{}".format(cls.root().name, cls.name) + if class_name_with_module in ipa_class_members: + fake_class(cls, ipa_class_members[class_name_with_module]) + +MANAGER.register_transform(scoped_nodes.Class, fix_ipa_classes) diff --git a/pylintrc b/pylintrc new file mode 100644 index 000000000..29db1049c --- /dev/null +++ b/pylintrc @@ -0,0 +1,85 @@ +[MASTER] +# Pickle collected data for later comparisons. +persistent=no + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins=pylint_plugins + +# Use multiple processes to speed up Pylint. +jobs=1 + +[MESSAGES CONTROL] + +enable= + all, + python3 + +disable= + R, + I, + invalid-name, + import-error, + abstract-method, + anomalous-backslash-in-string, + arguments-differ, + attribute-defined-outside-init, + bad-builtin, + bad-indentation, + bare-except, + broad-except, + dangerous-default-value, + eval-used, + exec-used, + fixme, + global-statement, + global-variable-not-assigned, + global-variable-undefined, + no-init, + pointless-except, + pointless-statement, + pointless-string-statement, + protected-access, + redefine-in-handler, + redefined-builtin, + redefined-outer-name, + reimported, + relative-import, + super-init-not-called, + undefined-loop-variable, + unnecessary-lambda, + unnecessary-semicolon, + unused-argument, + unused-variable, + useless-else-on-loop, + bad-classmethod-argument, + bad-continuation, + bad-mcs-classmethod-argument, + bad-mcs-method-argument, + bad-whitespace, + blacklisted-name, + invalid-name, + line-too-long, + missing-docstring, + multiple-imports, + multiple-statements, + old-style-class, + superfluous-parens, + too-many-lines, + unidiomatic-typecheck, + no-absolute-import + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=colorized + +# Tells whether to display a full report or only the messages +reports=no + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg})'