机器学习-零售商品销售预测（基于pyspark的7种回归预测，包含完整代码和数据）

import logging import sys import pandas as pd from pyspark.ml.evaluation import RegressionEvaluator from pyspark.ml.feature import VectorAssembler from pyspark.ml.regression import GBTRegressor from pyspark.ml.tuning import CrossValidator, ParamGridBuilder from pyspark.sql import SparkSession logging.basicConfig(level=logging.INFO, stream=sys.stdout) spark = SparkSession.builder.appName("big-mart-sales").getOrCreate() spark.sparkContext.setLogLevel('ERROR') train_df = spark.read.csv('train_model.csv', header=True, inferSchema=True) test_df = spark.read.csv('test_model.csv', header=True, inferSchema=True) # Get feature columns'name feature_cols = train_df.columns feature_cols.remove('Item_Outlet_Sales') # Combine all feature columns to one vectorAssembler = VectorAssembler(inputCols=feature_cols, outputCol='features') train_df = vectorAssembler.transform(train_df) train_df = train_df.select(['features', 'Item_Outlet_Sales']) test_df = vectorAssembler.transform(test_df) test_df = test_df.select(['features', 'Item_Outlet_Sales']) # Gradient-boosted Tree Regression model = GBTRegressor(featuresCol='features', labelCol='Item_Outlet_Sales', maxIter=10) evaluator = RegressionEvaluator(predictionCol="prediction", labelCol="Item_Outlet_Sales", metricName="rmse") paramGrid = ParamGridBuilder() \ .addGrid(model.maxDepth, [5, 10, 15]) \ .addGrid(model.minInstancesPerNode, [100, 150, 200]) \ .build() cv = CrossValidator(estimator=model, estimatorParamMaps=paramGrid, evaluator=evaluator, numFolds=5) cv_model = cv.fit(train_df) predictions = cv_model.transform(test_df) predictions.select("prediction", "Item_Outlet_Sales", "features").show(5) print("Root Mean Square Error (RMSE) on test data = %g" % evaluator.evaluate(predictions)) evaluator_r2 = RegressionEvaluator(predictionCol="prediction", labelCol="Item_Outlet_Sales", metricName="r2") print("R Squared (R2) on test data = %g" % evaluator_r2.evaluate(predictions)) # Don't support in TI-One # coef6 = pd.Series(cv_model.featureImportances, feature_cols).sort_values(ascending=False) # coef6.plot(kind='bar', title='Gradient-boosted Tree Feature Importances')