refine codes by flake8 (#2766)

Author: sneaxiy

python/couler/couler/__init__.py

@@ -11,4 +11,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._version import __version__
+from ._version import __version__  # noqa: F401

python/couler/couler/argo.py

@@ -14,6 +14,7 @@
 import atexit
 import copy
 import types
+import uuid
 from collections import OrderedDict
 
 import couler.pyfunc as pyfunc

python/couler/setup.py

@@ -43,7 +43,8 @@
 # The rest you shouldn't have to touch too much :)
 # ------------------------------------------------
 # Except, perhaps the License and Trove Classifiers!
-# If you do change the License, remember to change the Trove Classifier for that!
+# If you do change the License, remember to change
+# the Trove Classifier for that!
 
 here = os.path.abspath(os.path.dirname(__file__))
 

python/plotille_text_backend.py

@@ -14,8 +14,7 @@
 import io
 
 from matplotlib._pylab_helpers import Gcf
-from matplotlib.backend_bases import (FigureCanvasBase, FigureManagerBase,
-                                      RendererBase, _Backend)
+from matplotlib.backend_bases import FigureManagerBase
 from matplotlib.backends.backend_agg import FigureCanvasAgg, RendererAgg
 from PIL import Image
 from plotille import Canvas
@@ -89,7 +88,8 @@ def to_txt(self):
                 if center == (255, 255, 255):
                     continue
                 if x in range(1, w - 1) and y in range(1, h - 1):
-                    # Use the most deepest color in a 3x3 area as the center color
+                    # Use the most deepest color in a 3x3 area as
+                    # the center color
                     surrounding = i.getpixel((x, y - 1))  # upper
                     center = surrounding if grayscale(center) > grayscale(
                         surrounding) else center
@@ -140,7 +140,8 @@ def set_text(canvas, x, y, text):
 color_map = {}
 
 
-# Convert RGB to grayscale, see https://www.tutorialspoint.com/dip/grayscale_to_rgb_conversion.htm
+# Convert RGB to grayscale, see
+# https://www.tutorialspoint.com/dip/grayscale_to_rgb_conversion.htm
 def grayscale(rgb):
     return 0.3 * rgb[0] + 0.59 * rgb[1] + 0.11 * rgb[2]
 

python/sql_data.py

@@ -12,7 +12,7 @@
 # limitations under the License.
 
 import tensorflow as tf
-from MySQLdb import connect
+from MySQLdb import connect as mysql_connect
 
 
 def connect(user, passwd, host, port):
@@ -23,10 +23,11 @@ def connect(user, passwd, host, port):
         user: Specifies the MySQL user name.
         passwd : Specify the MySQL password.
         host : The host name or IP address.
-        port : Specifies the port number that attempts to connect to the MySQL server.
+        port : Specifies the port number that attempts to connect
+            to the MySQL server.
 
     """
-    return connect(user=user, passwd=passwd, host=host, port=port)
+    return mysql_connect(user=user, passwd=passwd, host=host, port=port)
 
 
 def load(db, slct, label, features):
@@ -49,10 +50,10 @@ def load(db, slct, label, features):
     f = [i[0] for i in cursor.description]  # get field names.
     c = list(zip(*cursor.fetchall()))  # transpose rows into columns.
     d = dict(zip(f, c))  # dict from field names to columns.
-    l = d.pop(label)
-    if features != None:
+    label = d.pop(label)
+    if features is not None:
         d = dict((k, d[k]) for k in features)
-    return d, l
+    return d, label
 
 
 def feature_columns(features):

python/test_magic.py

@@ -11,9 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
 import unittest
-from io import StringIO
 
 from IPython import get_ipython
 
@@ -23,16 +21,16 @@
 class TestSQLFlowMagic(unittest.TestCase):
     import random
     random.seed()
-    temp_database = "e2e_{}".format(random.randint(0, 1 << 32))
+    tmp_db = "e2e_{}".format(random.randint(0, 1 << 32))
     # the standard SQL statement
     create_database_statement = "create database if not exists {}".format(
-        temp_database)
-    create_statement = "create table {}.test_table_float_fea (features float, label int)".format(
-        temp_database)
-    insert_statement = "insert into {}.test_table_float_fea (features,label) values(1.0, 0), (2.0, 1)".format(
-        temp_database)
+        tmp_db)
+    create_statement = "create table {}.test_table_float_fea " \
+                       "(features float, label int)".format(tmp_db)
+    insert_statement = "insert into {}.test_table_float_fea (features,label)" \
+                       " values(1.0, 0), (2.0, 1)".format(tmp_db)
     select_statement = "select * from {}.test_table_float_fea limit 1;".format(
-        temp_database)
+        tmp_db)
 
     def setUp(self):
         ipython.run_cell_magic("sqlflow", "", self.create_database_statement)

python/test_magic_elasticdl.py

@@ -11,9 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
 import unittest
-from io import StringIO
 
 from IPython import get_ipython
 
@@ -22,39 +20,41 @@
 
 class TestSQLFlowMagic(unittest.TestCase):
     train_statement = """SELECT * FROM iris.train
-		TO TRAIN ElasticDLKerasClassifier 
-		WITH
-			model.num_classes = 10,
-			train.shuffle = 120,
-			train.epoch = 2,
-			train.grads_to_wait = 2,
-			train.tensorboard_log_dir = "",
-			train.checkpoint_steps = 0,
-			train.checkpoint_dir = "",
-			train.keep_checkpoint_max = 0,
-			eval.steps = 0,
-			eval.start_delay_secs = 100,
-			eval.throttle_secs = 0,
-			eval.checkpoint_filename_for_init = "",
-			engine.docker_image_prefix = "",
-			engine.master_resource_request = "cpu=1,memory=4096Mi,ephemeral-storage=10240Mi",
-			engine.worker_resource_request = "cpu=1,memory=4096Mi,ephemeral-storage=10240Mi",
-			engine.minibatch_size = 10,
-			engine.num_workers = 2,
-			engine.volume = "",
-			engine.image_pull_policy = "Always",
-			engine.restart_policy = "Never",
-			engine.extra_pypi_index = "",
-			engine.namespace = "default",
-			engine.master_pod_priority = "",
-			engine.cluster_spec = "",
-			engine.num_minibatches_per_task = 10,
-			engine.docker_image_repository = "",
-			engine.envs = ""
-		COLUMN
-			sepal_length, sepal_width, petal_length, petal_width
-		LABEL class
-		INTO trained_elasticdl_keras_classifier;
+TO TRAIN ElasticDLKerasClassifier
+WITH
+    model.num_classes = 10,
+    train.shuffle = 120,
+    train.epoch = 2,
+    train.grads_to_wait = 2,
+    train.tensorboard_log_dir = "",
+    train.checkpoint_steps = 0,
+    train.checkpoint_dir = "",
+    train.keep_checkpoint_max = 0,
+    eval.steps = 0,
+    eval.start_delay_secs = 100,
+    eval.throttle_secs = 0,
+    eval.checkpoint_filename_for_init = "",
+    engine.docker_image_prefix = "",
+    engine.master_resource_request =
+        "cpu=1,memory=4096Mi,ephemeral-storage=10240Mi",
+    engine.worker_resource_request =
+        "cpu=1,memory=4096Mi,ephemeral-storage=10240Mi",
+    engine.minibatch_size = 10,
+    engine.num_workers = 2,
+    engine.volume = "",
+    engine.image_pull_policy = "Always",
+    engine.restart_policy = "Never",
+    engine.extra_pypi_index = "",
+    engine.namespace = "default",
+    engine.master_pod_priority = "",
+    engine.cluster_spec = "",
+    engine.num_minibatches_per_task = 10,
+    engine.docker_image_repository = "",
+    engine.envs = ""
+COLUMN
+    sepal_length, sepal_width, petal_length, petal_width
+LABEL class
+INTO trained_elasticdl_keras_classifier;
 """
 
     def test_elasticdl(self):

python/test_sql_data.py

@@ -13,33 +13,35 @@
 
 import unittest
 
+import runtime.testing as testing
 import sql_data
-from runtime.db_test import testing_mysql_cfg
+from runtime.db import parseMySQLDSN
 
 
 class TestSQLData(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(TestSQLData, self).__init__(*args, **kwargs)
-        user, password, host, port, database = testing_mysql_cfg()
-        self.db = sql_data.connect(user, password, host, int(port))
+        dsn = testing.get_mysql_dsn()
+        user, passwd, host, port, _, _ = parseMySQLDSN(dsn)
+        self.db = sql_data.connect(user, passwd, host, int(port))
         self.assertIsNotNone(self.db)
 
     def test_load(self):
-        f, l = sql_data.load(self.db, 'SELECT * FROM iris.train LIMIT 3',
-                             'class', None)
+        f, label = sql_data.load(self.db, 'SELECT * FROM iris.train LIMIT 3',
+                                 'class', None)
         self.assertEqual(4, len(f.keys()))  # 4 features
-        self.assertEqual(3, len(l))  # label column length
+        self.assertEqual(3, len(label))  # label column length
 
     def test_load_with_filter(self):
         fs = ['sepal_length', 'petal_width']
-        f, l = sql_data.load(self.db, 'SELECT * FROM iris.train LIMIT 3',
-                             'class', fs)
+        f, label = sql_data.load(self.db, 'SELECT * FROM iris.train LIMIT 3',
+                                 'class', fs)
         self.assertEqual(len(fs), len(f))
-        self.assertEqual(3, len(l))  # label column length
+        self.assertEqual(3, len(label))  # label column length
 
     def test_feature_columns(self):
-        f, l = sql_data.load(self.db, 'SELECT * FROM iris.train LIMIT 3',
-                             'class', None)
+        f, label = sql_data.load(self.db, 'SELECT * FROM iris.train LIMIT 3',
+                                 'class', None)
         c = sql_data.feature_columns(f)
         self.assertEqual(4, len(c))  # 4 features
 

scripts/pre-commit/copyright.py

@@ -13,11 +13,8 @@
 
 import argparse
 import io
-import os
-import platform
 import re
 import subprocess
-import sys
 
 COPYRIGHT = '''
 Copyright 2020 The SQLFlow Authors. All rights reserved.
@@ -42,7 +39,7 @@
 
 NEW_LINE_MARK = '\n'
 COPYRIGHT_HEADER = COPYRIGHT.split(NEW_LINE_MARK)[1]
-p = re.search('(\d{4})', COPYRIGHT_HEADER).group(0)
+p = re.search('(\d{4})', COPYRIGHT_HEADER).group(0)  # noqa: W605
 process = subprocess.Popen(["date", "+%Y"], stdout=subprocess.PIPE)
 date, err = process.communicate()
 date = date.decode("utf-8").rstrip("\n")
@@ -59,7 +56,8 @@ def generate_copyright(template, lang='go'):
     BLANK = " "
     ans = LANG_COMMENT_MARK + BLANK + COPYRIGHT_HEADER + NEW_LINE_MARK
     for lino, line in enumerate(lines):
-        if lino == 0 or lino == 1 or lino == len(lines) - 1: continue
+        if lino == 0 or lino == 1 or lino == len(lines) - 1:
+            continue
         if len(line) == 0:
             BLANK = ""
         else:
@@ -99,16 +97,19 @@ def main(argv=None):
         second_line = fd.readline()
         third_line = fd.readline()
         # check for 3 head lines
-        if "COPYRIGHT " in first_line.upper(): continue
-        if "COPYRIGHT " in second_line.upper(): continue
-        if "COPYRIGHT " in third_line.upper(): continue
+        if "COPYRIGHT " in first_line.upper():
+            continue
+        if "COPYRIGHT " in second_line.upper():
+            continue
+        if "COPYRIGHT " in third_line.upper():
+            continue
         skip_one = False
         skip_two = False
         if first_line.startswith("#!"):
             skip_one = True
-        if PYTHON_ENCODE.match(second_line) != None:
+        if PYTHON_ENCODE.match(second_line) is not None:
             skip_two = True
-        if PYTHON_ENCODE.match(first_line) != None:
+        if PYTHON_ENCODE.match(first_line) is not None:
             skip_one = True
 
         original_content_lines = io.open(filename,