Merge pull request #1 from Nosfe/patch-1

Reading hadoopConfiguration directly from SparkContext.
This commit is contained in:
Yin Huai 2015-07-22 10:27:24 -07:00
commit b10fa582ea

View File

@ -143,7 +143,7 @@ abstract class Dataset(
def checkData(): Unit = {
tablesForTest.foreach { table =>
val fs = FileSystem.get(new java.net.URI(table.outputDir), new Configuration())
val fs = FileSystem.get(new java.net.URI(table.outputDir), sparkContext.hadoopConfiguration)
val exists = fs.exists(new Path(table.outputDir))
val wasSuccessful = fs.exists(new Path(s"${table.outputDir}/_SUCCESS"))
@ -302,4 +302,4 @@ abstract class Dataset(
}
new ExperimentStatus
}
}
}