adding linear regression line on multiple scatter subplots, coloured differently depends on slope being positive or negative

Question

So, I'm quite lost in how to retrieve my x and y for calling the polyfit function.

as the question states, my attempt is on each subplot my code produces:

to draw a linear regression, having, let's say, red colour if the slope is negative, or green if positive.

my code for drawing the figure is:

def show_monthly_temp(tmax):
tmax_grouped_avg = tmax.groupby(tmax.index.strftime("%m/%Y")).mean()
tmax_grouped_avg['datetime'] = pd.to_datetime(tmax_grouped_avg.index)
tmax_grouped_avg['Year'] = tmax_grouped_avg['datetime'].dt.year
groups = tmax_grouped_avg.sort_values('datetime').groupby(tmax_grouped_avg['datetime'].dt.month)
groups_df = pd.DataFrame(groups)
groups_df.to_csv("gaga")
f, axes = plt.subplots(nrows=3, ncols=4, figsize=(12, 6))

for (grp_id, grp_df), ax in zip(groups, axes.ravel()):
    print(grp_id)
    grp_df.plot.scatter(ax=ax, x='Year', y='TMAX', title=f'{calendar.month_name[grp_id]}', legend=False,
                        sharey=False, sharex=False)

plt.suptitle('Maximum temperature for each month')
plt.tight_layout()
plt.show()

and just in case, the whole code is:

import calendar

import pandas as pd
from datetime import datetime
import numpy as np
import scipy as stats
import matplotlib.pyplot as plt


def show_monthly_temp(tmax):
    tmax_grouped_avg = tmax.groupby(tmax.index.strftime("%m/%Y")).mean()
    tmax_grouped_avg['datetime'] = pd.to_datetime(tmax_grouped_avg.index)
    tmax_grouped_avg['Year'] = tmax_grouped_avg['datetime'].dt.year
    groups = tmax_grouped_avg.sort_values('datetime').groupby(tmax_grouped_avg['datetime'].dt.month)
    groups_df = pd.DataFrame(groups)
    groups_df.to_csv("gaga")
    f, axes = plt.subplots(nrows=3, ncols=4, figsize=(12, 6))

    for (grp_id, grp_df), ax in zip(groups, axes.ravel()):
        print(grp_id)
        grp_df.plot.scatter(ax=ax, x='Year', y='TMAX', title=f'{calendar.month_name[grp_id]}', legend=False,
                            sharey=False, sharex=False)

    plt.suptitle('Maximum temperature for each month')
    plt.tight_layout()
    plt.show()


def show_snow_days(snow):
    # transform to a list
    monthsDataFrames = [snow[snow.apply(lambda d: d.index.month == month)].dropna() for month in range(1, 13)]
    ax = plt.subplot(111)
    for i in range(len(monthsDataFrames)):
        ax.boxplot(monthsDataFrames[i].values, positions=[i])
    plt.xticks([i for i in range(12)], [str(month) for month in range(1, 13)])
    plt.show()


if __name__ == '__main__':
    df = pd.read_csv("2961941.csv")

    # set date column as index, drop the 'DATE' column to avoid repititions + create as datetime object
    # speed up parsing using infer_datetime_format=True.
    df.index = pd.to_datetime(df['DATE'], infer_datetime_format=True)

    # create new tables

    tmax = df.filter(['TMAX'], axis=1).dropna()
    snow = df.filter(['SNOW']).dropna()

    # count number of snow day samples - make sure at least >= 28
    snow_grouped = snow.groupby(pd.Grouper(level='DATE', freq="M")).transform('count')
    snow = (snow[snow_grouped['SNOW'] >= 28])

    # count number of tmax day samples - make sure at least >= 28
    tmax_grouped = tmax.groupby(pd.Grouper(level='DATE', freq="M")).transform('count')
    tmax = (tmax[tmax_grouped['TMAX'] >= 28])

    ################ Until here - initialized data ###############
    show_monthly_temp(tmax)
    show_snow_days(snow)

thanks! :-)

Answer 1

from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import calendar

df = pd.read_csv('dataForInquirer.csv',
                 index_col='DATE',
                 parse_dates=True)


def show_monthly_temp(tmax):
    tmax_grouped_avg = tmax.groupby(tmax.index.strftime("%m/%Y")).mean()
    tmax_grouped_avg['datetime'] = pd.to_datetime(tmax_grouped_avg.index)
    tmax_grouped_avg['Year'] = tmax_grouped_avg['datetime'].dt.year
    groups = tmax_grouped_avg.sort_values('datetime').groupby(tmax_grouped_avg['datetime'].dt.month)
    groups_df = pd.DataFrame(groups)
    groups_df.to_csv("gaga")
    f, axes = plt.subplots(nrows=3, ncols=4, figsize=(12, 6))

    for (grp_id, grp_df), ax in zip(groups, axes.ravel()):
        grp_df.plot.scatter(ax=ax, x='Year', y='TMAX', title=f'{calendar.month_name[grp_id]}', legend=False,
                            sharey=False, sharex=False)
        linear_regressor = LinearRegression()
        ind = np.array(grp_df['Year']).reshape((-1, 1))

        linear_regressor.fit(ind, grp_df['TMAX'])
        lr = linear_regressor.predict(ind)
        if lr[-1] <= lr[0]:
            ax.plot(grp_df['Year'], lr, color='red')
        else:
            ax.plot(grp_df['Year'], lr, color='green')



show_monthly_temp(df)
plt.suptitle('Maximum temperature for each month')
plt.tight_layout()
plt.show()

To calculate the linear regression, I used the 'sklearn' library. In which the indexes on the x axis and the values themselves are fed. Further, the values of the target are predicted by the indices. If the line has a negative slope or is horizontal, then it will be red, otherwise green. I checked it works on my data. Substitute your data into the 'show_monthly_temp' function, I left df (don't forget to change it). my dataframe looks like this:

            FLTO
DATE                 
1943-04-02   -0.08200
1943-04-30   -0.05600
1943-05-31   -0.05000
1943-06-30   -0.04700
1943-07-31   -0.02000
...               ...
2020-01-30    0.32707
2020-03-01   -1.54160
2020-04-01   -3.17358

how do get data frame:

df = pd.read_csv('name_file.csv',
                     index_col='DATE',
                     parse_dates=True,
                     infer_datetime_format=True)