odps/mars_extension/legacy/core.py [106:197]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - with_notebook=False, notebook_cpu=None, notebook_mem=None, with_graphscope=False, coordinator_cpu=None, coordinator_mem=None, timeout=None, extra_modules=None, resources=None, instance_id=None, name="default", if_exists="reuse", project=None, **kw ): """ Create a Mars cluster and a Mars session as default session, then all tasks will be submitted to cluster. :param worker_num: mars cluster worker's number :param worker_cpu: number of cpu cores on each mars worker :param worker_mem: memory size on each mars worker :param cache_mem: cache memory size on each mars worker :param disk_num: number of mounted disk :param min_worker_num: return if cluster worker's number reach to min_worker :param resources: resources name :param extra_modules: user defined module path :param scheduler_num: the number of schedulers, default is 0 :param with_notebook: whether launch jupyter notebook, defaullt is False :param instance_id: existing mars cluster's instance id :param name: cluster name, 'default' will be default name :param if_exists: 'reuse', 'raise' or 'ignore', if 'reuse', will reuse the first created cluster with the same name, if not created, create a new one; if 'raise', will fail if cluster with same name created already; if 'ignore', will always create a new cluster :param project: project name :return: class: `MarsClient` """ from .deploy.client import MarsCupidClient if kw.get("proxy_endpoint", None) is not None: cupid_options.cupid.proxy_endpoint = kw["proxy_endpoint"] if if_exists not in ("reuse", "raise", "ignore", "restart"): raise ValueError('`if_exists` should be "reuse", "raise, "ignore" or "restart"') if min_worker_num is not None and min_worker_num > worker_num: raise ValueError("`min_worker` cannot greater than `worker_num`") task_name = "MARS_TASK_{}".format(hashlib.md5(to_binary(name)).hexdigest()) _check_internal(odps.endpoint) if instance_id is not None: inst = odps.get_instance(instance_id, project=project) client = MarsCupidClient(odps, inst, project=project) elif if_exists in ("reuse", "raise", "restart"): client = None # need to check the instances before for prev_task_name, prev_instance in list_mars_instances( odps, project=project, days=2, return_task_name=True ): if prev_task_name == task_name: # found a instance with the same task name if if_exists in ("reuse", "restart"): if if_exists == "reuse": logger.info( "Reusing existing Mars cluster({}), logview address: \n{}".format( name, prev_instance.get_logview_address() ) ) client = MarsCupidClient(odps, prev_instance, project=project) break else: assert if_exists == "raise" raise ValueError('Cluster("{}") exists'.format(name)) if if_exists == "restart" and client is not None: # if exists Mars cluster, stop it first client.stop_server() client = None if client is None: # not create before, just create a new one client = MarsCupidClient(odps, project=project) else: client = MarsCupidClient(odps, project=project) worker_mem = int(worker_mem * 1024**3) cache_mem = int(cache_mem * 1024**3) if cache_mem else None disk_size = int(disk_size * 1024**3) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - odps/mars_extension/oscar/core.py [100:192]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - with_notebook=False, notebook_cpu=None, notebook_mem=None, with_graphscope=False, coordinator_cpu=None, coordinator_mem=None, timeout=None, extra_modules=None, resources=None, instance_id=None, name="default", if_exists="reuse", project=None, **kw ): """ Create a Mars cluster and a Mars session as default session, then all tasks will be submitted to cluster. :param worker_num: mars cluster worker's number :param worker_cpu: number of cpu cores on each mars worker :param worker_mem: memory size on each mars worker :param cache_mem: cache memory size on each mars worker :param disk_num: number of mounted disk :param min_worker_num: return if cluster worker's number reach to min_worker :param resources: resources name :param extra_modules: user defined module path :param supervisor_num: the number of supervisors, default is 0 :param with_notebook: whether launch jupyter notebook, default is False :param instance_id: existing mars cluster's instance id :param name: cluster name, 'default' will be default name :param if_exists: 'reuse', 'raise' or 'ignore', if 'reuse', will reuse the first created cluster with the same name, if not created, create a new one; if 'raise', will fail if cluster with same name created already; if 'ignore', will always create a new cluster :param project: project name :return: class: `MarsClient` """ from cupid.config import options as cupid_options from .deploy.client import MarsCupidClient if kw.get("proxy_endpoint", None) is not None: cupid_options.cupid.proxy_endpoint = kw["proxy_endpoint"] if if_exists not in ("reuse", "raise", "ignore", "restart"): raise ValueError('`if_exists` should be "reuse", "raise, "ignore" or "restart"') if min_worker_num is not None and min_worker_num > worker_num: raise ValueError("`min_worker` cannot greater than `worker_num`") task_name = "MARS_TASK_{}".format(hashlib.md5(to_binary(name)).hexdigest()) _check_internal(odps.endpoint) if instance_id is not None: inst = odps.get_instance(instance_id, project=project) client = MarsCupidClient(odps, inst, project=project) elif if_exists in ("reuse", "raise", "restart"): client = None # need to check the instances before for prev_task_name, prev_instance in list_mars_instances( odps, project=project, days=2, return_task_name=True ): if prev_task_name == task_name: # found a instance with the same task name if if_exists in ("reuse", "restart"): if if_exists == "reuse": logger.info( "Reusing existing Mars cluster({}), logview address: \n{}".format( name, prev_instance.get_logview_address() ) ) client = MarsCupidClient(odps, prev_instance, project=project) break else: assert if_exists == "raise" raise ValueError('Cluster("{}") exists'.format(name)) if if_exists == "restart" and client is not None: # if exists Mars cluster, stop it first client.stop_server() client = None if client is None: # not create before, just create a new one client = MarsCupidClient(odps, project=project) else: client = MarsCupidClient(odps, project=project) worker_mem = int(worker_mem * 1024**3) cache_mem = int(cache_mem * 1024**3) if cache_mem else None disk_size = int(disk_size * 1024**3) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -