Skip to content

Commit 1d041d8

Browse files
authored
Cleanup submitter code (#2847)
* Remove hdfs params in Python * fomat code * Cleanup submitter code
1 parent ee28761 commit 1d041d8

3 files changed

Lines changed: 7 additions & 23 deletions

File tree

python/runtime/alisa/submitter.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ def submit_alisa_train(datasource, estimator_string, select, validation_select,
161161
162162
Args:
163163
datasource: string
164-
Like: odps://access_id:access_key@service.com/api?
164+
Like: alisa://access_id:access_key@service.com/api?
165165
curr_project=test_ci&scheme=http
166166
estimator_string: string
167167
Tensorflow estimator name, Keras class name, or XGBoost

python/runtime/pai/submitter.py

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -132,18 +132,15 @@ def get_oss_model_url(model_full_path):
132132
return "oss://%s/%s" % (oss.SQLFLOW_MODELS_BUCKET, model_full_path)
133133

134134

135-
def parse_maxcompute_dsn(datasource):
136-
return MaxComputeConnection.get_uri_parts(datasource)
137-
138-
139135
def drop_pai_model(datasource, model_name):
140136
"""Drop PAI model
141137
142138
Args:
143139
datasource: current datasource
144140
model_name: name of the model to drop
145141
"""
146-
user, passwd, address, database = parse_maxcompute_dsn(datasource)
142+
user, passwd, address, database = MaxComputeConnection.get_uri_parts(
143+
datasource)
147144
cmd = "drop offlinemodel if exists %s" % model_name
148145
subprocess.run([
149146
"odpscmd", "-u", user, "-p", passwd, "--project", database,
@@ -220,7 +217,8 @@ def submit_pai_task(pai_cmd, datasource):
220217
pai_cmd: The command to submit
221218
datasource: The datasource this cmd will manipulate
222219
"""
223-
user, passwd, address, project = parse_maxcompute_dsn(datasource)
220+
user, passwd, address, project = MaxComputeConnection.get_uri_parts(
221+
datasource)
224222
cmd = [
225223
"odpscmd", "--instance-priority", "9", "-u", user, "-p", passwd,
226224
"--project", project, "--endpoint", address, "-e", pai_cmd
@@ -234,22 +232,18 @@ def submit_pai_task(pai_cmd, datasource):
234232
def get_oss_model_save_path(datasource, model_name):
235233
if not model_name:
236234
return None
237-
user, _, _, project = parse_maxcompute_dsn(datasource)
235+
user, _, _, project = MaxComputeConnection.get_uri_parts(datasource)
238236
user = user or "unknown"
239237
return "/".join([project, user, model_name])
240238

241239

242-
def get_datasource_dsn(datasource):
243-
return datasource.split("://")[1]
244-
245-
246240
def get_project(datasource):
247241
"""Get the project info from given datasource
248242
249243
Args:
250244
datasource: The odps url to extract project
251245
"""
252-
_, _, _, project = parse_maxcompute_dsn(datasource)
246+
_, _, _, project = MaxComputeConnection.get_uri_parts(datasource)
253247
return project
254248

255249

python/runtime/pai/submitter_test.py

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,6 @@ def test_get_oss_model_url(self):
2727
url = submitter.get_oss_model_url("user_a/model")
2828
self.assertEqual("oss://sqlflow-models/user_a/model", url)
2929

30-
def test_get_datasource_dsn(self):
31-
ds = "odps://access_id:access_key@service.com/api?" \
32-
"curr_project=test_ci&scheme=http"
33-
expected_dsn = "access_id:access_key@service.com/api?" \
34-
"curr_project=test_ci&scheme=http"
35-
dsn = submitter.get_datasource_dsn(ds)
36-
self.assertEqual(expected_dsn, dsn)
37-
project = "test_ci"
38-
self.assertEqual(project, submitter.get_project(ds))
39-
4030
def test_get_pai_tf_cmd(self):
4131
conf = get_cluster_config({})
4232
os.environ[

0 commit comments

Comments
 (0)