in src/main/scala/com/aliyun/emr/example/spark/SparkMaxComputeDemo.scala [27:70]
def main(args: Array[String]): Unit = {
if (args.length < 6) {
System.err.println(
"""Usage: SparkMaxComputeDemo <accessKeyId> <accessKeySecret> <envType> <project> <table> <numPartitions>
|
|Arguments:
|
| accessKeyId Aliyun Access Key ID.
| accessKeySecret Aliyun Key Secret.
| envType 0 or 1
| 0: Public environment.
| 1: Aliyun internal environment, i.e. Aliyun ECS etc.
| project Aliyun ODPS project
| table Aliyun ODPS table
| numPartitions the number of RDD partitions
""".stripMargin)
System.exit(1)
}
val accessKeyId = args(0)
val accessKeySecret = args(1)
val envType = args(2).toInt
val project = args(3)
val table = args(4)
val numPartitions = args(5).toInt
val urls = Seq(
Seq("http://service.odps.aliyun.com/api", "http://dt.odps.aliyun.com"), // public environment
Seq("http://odps-ext.aliyun-inc.com/api", "http://dt-ext.odps.aliyun-inc.com") // Aliyun internal environment
)
val conf = new SparkConf().setAppName("E-MapReduce Demo 3-1: Spark MaxCompute Demo (Scala)")
val sc = new SparkContext(conf)
val odpsOps = envType match {
case 0 =>
OdpsOps(sc, accessKeyId, accessKeySecret, urls(0)(0), urls(0)(1))
case 1 =>
OdpsOps(sc, accessKeyId, accessKeySecret, urls(1)(0), urls(1)(1))
}
val odpsData = odpsOps.readTable(project, table, read, numPartitions)
println(s"Count (odpsData): ${odpsData.count()}")
}