Upload 40 files

nt_exp.py

@@ -0,0 +1,203 @@
+import os
+
+import numpy as np
+import pandas as pd
+from scipy.io import arff
+from sklearn import preprocessing
+from sklearn.model_selection import train_test_split
+
+from nt_gan import GAN
+from nt_gg import GG
+
+dataset_directory = 'datasets'
+saved_models_path = 'outputs'
+
+
+def prepare_architecture(arff_data_path):
+    """
+    This function create the architecture of the GAN network.
+    The generator and the discriminator are created and then combined into the GAN model
+    :param arff_data_path: data path for the arff file
+    :return: a dictionary with all the relevant variables for the next stages
+    """
+    data, meta_data = arff.loadarff(arff_data_path)  # This function reads arff file into tuple of data and its meta.
+    df = pd.DataFrame(data)
+    columns = df.columns
+    transformed_data, x, x_scaled, meta_data_rev, min_max_scaler = create_scaled_data(df, meta_data)
+
+    number_of_features = len(transformed_data.columns)  # Define the GAN and training parameters
+
+    return x_scaled, meta_data_rev, columns, min_max_scaler, number_of_features
+
+
+def create_scaled_data(df, meta_data):
+    """
+
+    :param df:
+    :param meta_data:
+    :return:
+    """
+    meta_data_dict = {k: {a.replace(' ', ''): b + 1 for b, a in enumerate(v.values)} for k, v in
+                      meta_data._attributes.items() if
+                      v.type_name != 'numeric'}  # Starts from one and not zero because one is for Nan values
+    meta_data_rev = {k: {b + 1: a.replace(' ', '') for b, a in enumerate(v.values)} for k, v in
+                     meta_data._attributes.items() if
+                     v.type_name != 'numeric'}  # Starts from one and not zero because one is for Nan values
+    transformed_data = df.copy()
+    for col in df.columns:
+        if col in meta_data_dict:
+            # Sometimes the values can not be found in the meta data, so we treat these values as Nan
+            transformed_data[col] = transformed_data[col].apply(
+                lambda x: meta_data_dict[col][str(x).split('\'')[1]] if str(x).split('\'')[1] in meta_data_dict[
+                    col] else 0)
+    x = transformed_data.values  # returns a numpy array
+    min_max_scaler = preprocessing.MinMaxScaler()
+    x_scaled = min_max_scaler.fit_transform(x)
+    return transformed_data, x, x_scaled, meta_data_rev, min_max_scaler
+
+
+def re_scaled_data(data, columns, meta_data_rev, min_max_scaler):
+    """
+    This function re-scaled the fake data to the original format.
+    :param data: the data we want to re scaled
+    :param columns:
+    :param meta_data_rev:
+    :return:
+    """
+    data_inv = min_max_scaler.inverse_transform(data)
+    df = pd.DataFrame(data_inv, columns=columns)
+    transformed_data = df.copy()
+    for col in transformed_data.columns:
+        if col in meta_data_rev:
+            # Sometimes the values can not be found in the meta data, so we treat these values as Nan
+            transformed_data[col] = transformed_data[col].apply(
+                lambda x: meta_data_rev[col][int(round(x))] if int(round(x)) in meta_data_rev[
+                    col] else np.nan)
+    return transformed_data
+
+
+def first_question():
+    """
+    This function answers the first question
+    :return:
+    """
+    to_plot_losses = True
+    results_output = os.path.join(saved_models_path, f'question_one_results.csv')
+    results = {'dataset': [], 'lr': [], 'ep': [], 'bs': [], 'alpha': [], 'dropout': [], 'gen_loss': [], 'dis_loss': [],
+               'activation': [], 'fooled_len': [], 'not_fooled_len': [], 'mean_min_distance_fooled': [],
+               'mean_min_distance_not_fooled': [], 'mean_min_distance_gap': []}
+    # w1 * (MMDF + MMDNF) - w3 * (MMDG) + w2 * (NFL/ 100)
+    # MMDG = MMDNF - MMDF
+    # data_name = ["adult", "bank-full"]
+    # learning_rate = [0.01, 0.001, 0.0001]
+    # epochs = [5, 10, 15]
+    # batch_size = [64, 128, 1024]
+    # alpha_relu = [0.2, 0.5]
+    # dropout = [0.3, 0.5]
+    data_name = ["adult"]
+    learning_rate = [0.001]
+    epochs = [10]
+    batch_size = [128]
+    alpha_relu = [0.5]
+    dropout = [0.5]
+    loss = 'binary_crossentropy'
+    activation = 'sigmoid'
+
+    for data in data_name:
+        for lr in learning_rate:
+            for ep in epochs:
+                for bs in batch_size:
+                    for al in alpha_relu:
+                        for dr in dropout:
+                            arff_data_path = f'./datasets/{data}.arff'
+                            model_name = f'data_{data}_ep_{ep}_bs_{bs}_lr_{lr}_al_{al}_dr_{dr}'
+                            pca_output = os.path.join(saved_models_path, f'{model_name}_pca.png')
+                            fooled_output = os.path.join(saved_models_path, f'{model_name}_fooled.csv')
+                            not_fooled_output = os.path.join(saved_models_path, f'{model_name}_not_fooled.csv')
+
+                            x_scaled, meta_data_rev, columns, min_max_scaler, number_of_features = prepare_architecture(
+                                arff_data_path)
+                            gan_obj = GAN(number_of_features=number_of_features, saved_models_path=saved_models_path,
+                                          learning_rate=lr, alpha_relu=al, dropout=dr,
+                                          loss=loss, activation=activation)
+                            gen_loss, dis_loss = gan_obj.train(scaled_data=x_scaled, epochs=ep, batch_size=bs,
+                                                               to_plot_losses=to_plot_losses, model_name=model_name)
+                            dis_fooled_scaled, dis_not_fooled_scaled, mean_min_distance_fooled, mean_min_distance_not_fooled = gan_obj.test(
+                                scaled_data=x_scaled, sample_num=100, pca_output=pca_output)
+                            dis_fooled = re_scaled_data(data=dis_fooled_scaled, columns=columns,
+                                                        meta_data_rev=meta_data_rev,
+                                                        min_max_scaler=min_max_scaler)
+                            dis_fooled.to_csv(fooled_output)
+                            dis_not_fooled = re_scaled_data(data=dis_not_fooled_scaled, columns=columns,
+                                                            meta_data_rev=meta_data_rev,
+                                                            min_max_scaler=min_max_scaler)
+                            dis_not_fooled.to_csv(not_fooled_output)
+                            results['dataset'].append(data)
+                            results['lr'].append(lr)
+                            results['ep'].append(ep)
+                            results['bs'].append(bs)
+                            results['alpha'].append(al)
+                            results['dropout'].append(dr)
+                            results['gen_loss'].append(gen_loss)
+                            results['dis_loss'].append(dis_loss)
+                            results['activation'].append(activation)
+                            results['fooled_len'].append(len(dis_fooled_scaled))
+                            results['not_fooled_len'].append(len(dis_not_fooled_scaled))
+                            results['mean_min_distance_fooled'].append(mean_min_distance_fooled)
+                            results['mean_min_distance_not_fooled'].append(mean_min_distance_not_fooled)
+                            results['mean_min_distance_gap'].append(mean_min_distance_not_fooled-mean_min_distance_fooled)
+    results_df = pd.DataFrame.from_dict(results)
+    results_df.to_csv(results_output, index=False)
+
+
+def second_question():
+
+    data_name = ["adult", "bank-full"]
+    learning_rate = [0.001]
+    epochs = [10]
+    batch_size = [128]
+    alpha_relu = [0.2]
+    dropout = [0.3]
+    results = {'dataset': [], 'lr': [], 'ep': [], 'bs': [], 'alpha': [], 'dropout': [], 'gen_loss': [], 'proba_error': []}
+    combs = len(data_name) * len(learning_rate) * len(epochs) * len(batch_size) * len(alpha_relu) * len(dropout)
+    i = 1
+    for data in data_name:
+        for lr in learning_rate:
+            for ep in epochs:
+                for bs in batch_size:
+                    for al in alpha_relu:
+                        for dr in dropout:
+                            print(f'Running combination {i}/{combs}')
+                            data_path = f'./datasets/{data}.arff'
+                            model_name = f'data_{data}_ep_{ep}_bs_{bs}_lr_{lr}_part2'
+                            x_scaled, meta_data_rev, cols, min_max_scaler, feature_num = prepare_architecture(data_path)
+                            general_generator = GG(feature_num, saved_models_path, lr, dr, al)
+                            x_train, x_test, y_train, y_test = train_test_split(x_scaled[:, :-1], x_scaled[:, -1], test_size=0.1)
+                            general_generator.train_gg(x_train, y_train, ep, bs, model_name, data, saved_models_path, True)
+                            error = general_generator.get_error()
+                            results['dataset'].append(data)
+                            results['lr'].append(lr)
+                            results['ep'].append(ep)
+                            results['bs'].append(bs)
+                            results['alpha'].append(al)
+                            results['dropout'].append(dr)
+                            results['gen_loss'].append(general_generator.losses['gen_loss'][-1])
+                            results['proba_error'].append(error.mean())
+                            i += 1
+                            # Test set performance
+                            general_generator.plot_discriminator_results(x_test, y_test, data, saved_models_path)
+                            general_generator.plot_generator_results(data, saved_models_path)
+
+    results_output = os.path.join(saved_models_path, f'question_two_results.csv')
+    results_df = pd.DataFrame.from_dict(results)
+    # results_df.to_csv(results_output, index=False)
+
+
+
+def main():
+    # first_question()
+    second_question()
+
+
+if __name__ == '__main__':
+    main()

nt_gan.py

@@ -0,0 +1,333 @@
+import os
+from itertools import compress
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from keras.layers import Dense, Dropout, LeakyReLU
+from keras.models import Sequential
+from keras.optimizers import Adam
+from numpy.random import randn
+from sklearn.decomposition import PCA
+from tqdm import tqdm
+
+
+class GAN(object):
+    def __init__(self, number_of_features, saved_models_path, learning_rate, alpha_relu, dropout, loss, activation):
+        """
+        A constructor for the GAN class
+        :param number_of_features: number of features
+        :param saved_models_path: the output folder path
+        """
+        self.saved_models_path = saved_models_path
+        self.number_of_features = number_of_features
+
+        self.generator_model = None
+        self.noise_dim = None
+        self.discriminator_model = None
+        self.learning_rate = learning_rate
+        self.gan_model = None
+        self.activation = activation
+        self.alpha_relu = alpha_relu
+        self.loss = loss
+        self.dropout = dropout
+        self.number_of_features = number_of_features
+
+        self.build_generator()  # build the generator
+        self.build_discriminator()  # build the discriminator
+        self.build_gan()  # build the GAN
+
+    def build_generator(self):
+        """
+        This function creates the generator model
+        :return:
+        """
+        noise_size = int(self.number_of_features / 2)
+        self.noise_dim = (noise_size,)  # size of the noise space
+
+        self.generator_model = Sequential()
+        self.generator_model.add(Dense(int(self.number_of_features * 2), input_shape=self.noise_dim))
+        self.generator_model.add(LeakyReLU(alpha=self.alpha_relu))
+
+        self.generator_model.add(Dense(int(self.number_of_features * 4)))
+        self.generator_model.add(LeakyReLU(alpha=self.alpha_relu))
+        self.generator_model.add(Dropout(self.dropout))
+
+        self.generator_model.add(Dense(int(self.number_of_features * 2)))
+        self.generator_model.add(LeakyReLU(alpha=self.alpha_relu))
+        self.generator_model.add(Dropout(self.dropout))
+
+        # Compile it
+        self.generator_model.add(Dense(self.number_of_features, activation=self.activation))
+        self.generator_model.summary()
+
+    def build_discriminator(self):
+        """
+        Create discriminator model
+        :return:
+        """
+        self.discriminator_model = Sequential()
+
+        self.discriminator_model.add(Dense(self.number_of_features * 2, input_shape=(self.number_of_features,)))
+        self.discriminator_model.add(LeakyReLU(alpha=self.alpha_relu))
+
+        self.discriminator_model.add(Dense(self.number_of_features * 4))
+        self.discriminator_model.add(LeakyReLU(alpha=self.alpha_relu))
+        self.discriminator_model.add(Dropout(self.dropout))
+
+        self.discriminator_model.add(Dense(self.number_of_features * 2))
+        self.discriminator_model.add(LeakyReLU(alpha=self.alpha_relu))
+        self.discriminator_model.add(Dropout(self.dropout))
+
+        # Compile it
+        self.discriminator_model.add(Dense(1, activation=self.activation))
+        optimizer = Adam(lr=self.learning_rate)
+        self.discriminator_model.compile(loss=self.loss, optimizer=optimizer)
+        self.discriminator_model.summary()
+
+    def build_gan(self):
+        """
+        Create the GAN network
+        :return: the GAN model object
+        """
+        self.gan_model = Sequential()
+        self.discriminator_model.trainable = False
+
+        # The following lines connect the generator and discriminator models to the GAN.
+        self.gan_model.add(self.generator_model)
+        self.gan_model.add(self.discriminator_model)
+
+        # Compile it
+        optimizer = Adam(lr=self.learning_rate)
+        self.gan_model.compile(loss=self.loss, optimizer=optimizer)
+
+        return self.gan_model
+
+    def train(self, scaled_data, epochs, batch_size, to_plot_losses, model_name):
+        """
+        This function trains the generator and discriminator outputs
+        :param model_name:
+        :param to_plot_losses: whether or not to plot history
+        :param scaled_data: the data after min max scaling
+        :param epochs: number of epochs
+        :param batch_size: the batch size
+        :return: losses_list: returns the losses dictionary the generator or discriminator outputs
+        """
+        dis_output, gen_output, prev_output = self.check_for_existed_output(model_name)
+        if prev_output:
+            return -1, -1
+
+        losses_output = os.path.join(self.saved_models_path, f'{model_name}_losses.png')
+        discriminator_loss = []
+        generator_loss = []
+
+        # We need to use half of the batch size for the fake data and half for the real one
+        half_batch_size = int(batch_size / 2)
+        iterations = int(len(scaled_data) / half_batch_size)
+        iterations = iterations + 1 if len(scaled_data) % batch_size != 0 else iterations
+
+        for epoch in range(1, epochs + 1):  # iterates over the epochs
+            np.random.shuffle(scaled_data)
+            p_bar = tqdm(range(iterations), ascii=True)
+            for iteration in p_bar:
+                dis_loss, gen_loss = self.train_models(batch_size=batch_size, half_batch_size=half_batch_size,
+                                                       index=iteration, scaled_data=scaled_data)
+                discriminator_loss.append(dis_loss)
+                generator_loss.append(gen_loss)
+                p_bar.set_description(
+                    f"Epoch ({epoch}/{epochs}) | DISCRIMINATOR LOSS: {dis_loss:.2f} | GENERATOR LOSS: {gen_loss:.2f} |")
+
+        # Save weights for future use
+        self.discriminator_model.save_weights(dis_output)
+        self.generator_model.save_weights(gen_output)
+
+        # Plot losses
+        if to_plot_losses:
+            self.plot_losses(discriminator_loss=discriminator_loss, generator_loss=generator_loss,
+                             losses_output=losses_output)
+
+        return generator_loss[-1], discriminator_loss[-1]
+
+    def check_for_existed_output(self, model_name) -> (str, str, bool):
+        """
+        This function checks for existed output
+        :param model_name: model's name
+        :return:
+        """
+        prev_output = False
+        dis_output = os.path.join(self.saved_models_path, f'{model_name}_dis_weights.h5')
+        gen_output = os.path.join(self.saved_models_path, f'{model_name}_gen_weights.h5')
+        if os.path.exists(dis_output) and os.path.exists(gen_output):
+            print("The model was trained in the past")
+            self.discriminator_model.load_weights(dis_output)
+            self.generator_model.load_weights(gen_output)
+            prev_output = True
+        return dis_output, gen_output, prev_output
+
+    def train_models(self, batch_size, half_batch_size, index, scaled_data):
+        """
+        This function trains the discriminator and the generator
+        :param batch_size: batch size
+        :param half_batch_size: half of the batch size
+        :param index:
+        :param scaled_data:
+        :return:
+        """
+        self.discriminator_model.trainable = True
+
+        # Create a batch of real data and train the model
+        x_real, y_real = self.get_real_samples(data=scaled_data, batch_size=half_batch_size, index=index)
+        d_real_loss = self.discriminator_model.train_on_batch(x_real, y_real)
+
+        # Create a batch of fake data and train the model
+        x_fake, y_fake = self.create_fake_samples(batch_size=half_batch_size)
+        d_fake_loss = self.discriminator_model.train_on_batch(x_fake, y_fake)
+
+        avg_dis_loss = 0.5 * (d_real_loss + d_fake_loss)
+
+        # Create noise for the generator model
+        noise = randn(self.noise_dim[0] * batch_size).reshape((batch_size, self.noise_dim[0]))
+
+        self.discriminator_model.trainable = False
+        gen_loss = self.gan_model.train_on_batch(noise, np.ones((batch_size, 1)))
+
+        return avg_dis_loss, gen_loss
+
+    @staticmethod
+    def get_real_samples(data, batch_size, index):
+        """
+        Generate batch_size of real samples with class labels
+        :param data: the original data
+        :param batch_size: batch size
+        :param index: the index of the batch
+        :return: x: real samples, y: labels
+        """
+        start_index = batch_size * index
+        end_index = start_index + batch_size
+        x = data[start_index: end_index]
+
+        return x, np.ones((len(x), 1))
+
+    def create_fake_samples(self, batch_size):
+        """
+        Use the generator to generate n fake examples, with class labels
+        :param batch_size: batch size
+        :return:
+        """
+        noise = randn(self.noise_dim[0] * batch_size).reshape((batch_size, self.noise_dim[0]))
+        x = self.generator_model.predict(noise)  # create fake samples using the generator
+
+        return x, np.zeros((len(x), 1))
+
+    @staticmethod
+    def plot_losses(discriminator_loss, generator_loss, losses_output):
+        """
+        Plot training loss values
+        :param generator_loss:
+        :param discriminator_loss:
+        :param losses_output:
+        :return:
+        """
+        plt.plot(discriminator_loss)
+        plt.plot(generator_loss)
+        plt.xlabel('Iteration')
+        plt.ylabel('Loss')
+        plt.title('Discriminator and Generator Losses')
+        plt.legend(['Discriminator Loss', 'Generator Loss'])
+        plt.savefig(losses_output)
+
+    @staticmethod
+    def return_minimum_euclidean_distance(scaled_data, x):
+        """
+        This function returns the
+        :param scaled_data: the original data
+        :param x: a record we want to compare with
+        :return: the minimum distance and the index of the minimum value
+        """
+        s = np.power(np.power((scaled_data - np.array(x)), 2).sum(1), 0.5)
+        return pd.Series([s[s.argmin()], s.argmin()])
+
+    def test(self, scaled_data, sample_num, pca_output):
+        """
+        This function tests the model
+        :param scaled_data: the original scaled data
+        :param sample_num: number of samples to generate
+        :param pca_output: the output of PCA
+        :return:
+        """
+        x_fake, y_fake = self.create_fake_samples(batch_size=sample_num)
+        fake_pred = self.discriminator_model.predict(x_fake)
+
+        # Filter data to different matrices
+        dis_fooled_scaled = np.asarray(list(compress(x_fake, fake_pred > 0.5)))
+        dis_not_fooled_scaled = np.asarray(list(compress(x_fake, fake_pred <= 0.5)))
+
+        # ------------- Euclidean -------------
+        mean_min_distance_fooled, mean_min_distance_not_fooled = (-1, -1)
+        if len(dis_fooled_scaled) > 0 and len(dis_not_fooled_scaled) > 0:
+            mean_min_distance_fooled = self.get_mean_distance_score(scaled_data, dis_fooled_scaled)
+            print(f'The mean minimum distance for fooled samples is {mean_min_distance_fooled}')
+            mean_min_distance_not_fooled = self.get_mean_distance_score(scaled_data, dis_not_fooled_scaled)
+            print(f'The mean minimum distance for not fooled samples is {mean_min_distance_not_fooled}')
+        else:
+            print(f'The fooled xor the not Fooled data frames is empty')
+
+        # ------------- PCA --------------
+        data_pca_df = self.get_pca_df(scaled_data, 'original')
+        dis_fooled_pca_df = self.get_pca_df(dis_fooled_scaled, 'fooled')
+        dis_not_fooled_pca_df = self.get_pca_df(dis_not_fooled_scaled, 'not fooled')
+        pca_frames = [data_pca_df, dis_fooled_pca_df, dis_not_fooled_pca_df]
+        pca_result = pd.concat(pca_frames)
+        self.plot_pca(pca_result, pca_output)
+
+        return dis_fooled_scaled, dis_not_fooled_scaled, mean_min_distance_fooled, mean_min_distance_not_fooled
+
+    def get_mean_distance_score(self, scaled_data, dis_scaled):
+        """
+        This function returns the mean distance score for the given dataframe
+        :param scaled_data: the original data
+        :param dis_scaled: a dataframe
+        :return:
+        """
+        dis_fooled_scaled_ecu = pd.DataFrame(dis_scaled)
+        dis_fooled_scaled_ecu[['min_distance', 'similar_i']] = dis_fooled_scaled_ecu.apply(
+            lambda x: self.return_minimum_euclidean_distance(scaled_data, x), axis=1)
+        mean_min_distance_fooled = dis_fooled_scaled_ecu['min_distance'].mean()
+        return mean_min_distance_fooled
+
+    @staticmethod
+    def plot_pca(pca_result, pca_output):
+        """
+        This function plots the PCA figure
+        :param pca_result: dataframe with all the results
+        :param pca_output: output path
+        :return:
+        """
+        fig = plt.figure(figsize=(8, 8))
+        ax = fig.add_subplot(1, 1, 1)
+        ax.set_xlabel('Principal Component 1', fontsize=15)
+        ax.set_ylabel('Principal Component 2', fontsize=15)
+        ax.set_title('PCA With Two Components', fontsize=20)
+        targets = ['original', 'fooled', 'not fooled']
+        colors = ['r', 'g', 'b']
+        for target, color in zip(targets, colors):
+            indices_to_keep = pca_result['name'] == target
+            ax.scatter(pca_result.loc[indices_to_keep, 'comp1'], pca_result.loc[indices_to_keep, 'comp2'],
+                       c=color, s=50)
+        ax.legend(targets)
+        ax.grid()
+        plt.savefig(pca_output)
+
+    @staticmethod
+    def get_pca_df(scaled_data, data_name):
+        """
+        This function creates the PCA dataframe
+        :param scaled_data: the original data
+        :param data_name: the name of the column
+        :return:
+        """
+        pca = PCA(n_components=2)
+        principal_components = pca.fit_transform(scaled_data)
+        principal_df = pd.DataFrame(data=principal_components, columns=['comp1', 'comp2'])
+        principal_df['name'] = data_name
+        return principal_df

nt_gg.py

@@ -0,0 +1,282 @@
+import os
+import pickle
+import matplotlib.pyplot as plt
+import numpy as np
+from keras.layers import Dense, Dropout, LeakyReLU
+from keras.models import Sequential
+from keras.optimizers import Adam
+from numpy.random import randn
+from sklearn.ensemble import RandomForestClassifier
+from sklearn import metrics
+from tqdm import tqdm
+
+
+class GG(object):
+
+    def __init__(self, number_of_features, saved_models_path, learning_rate, dropout, alpha):
+        """
+        The constructor for the General Generator class.
+        :param number_of_features: Number of features in the data. Used to determine the noise dimensions
+        :param saved_models_path: The folder where we save the models.
+        """
+        self.saved_models_path = saved_models_path
+        self.number_of_features = number_of_features
+
+        self.generator_model = None
+        self.discriminator_model = RandomForestClassifier()
+        self.dropout = dropout
+        self.alpha = alpha
+        self.noise_dim = int(number_of_features / 2)
+        self.learning_rate = learning_rate
+        self.number_of_features = number_of_features
+        self.build_generator()  # build the generator.
+        self.losses = {'gen_loss': [], 'dis_loss_pred': [], 'dis_loss_proba': []}
+        # self.results = {}
+
+    def build_generator(self):
+        """
+        This function creates the generator model for the GG.
+        We used a fairly simple MLP architecture.
+        :return:
+        """
+
+        self.generator_model = Sequential()
+        self.generator_model.add(Dense(int(self.number_of_features * 2), input_shape=(self.noise_dim + 1, )))
+        self.generator_model.add(LeakyReLU(alpha=self.alpha))
+
+        self.generator_model.add(Dense(int(self.number_of_features * 4)))
+        self.generator_model.add(LeakyReLU(alpha=self.alpha))
+        self.generator_model.add(Dropout(self.dropout))
+
+        self.generator_model.add(Dense(int(self.number_of_features * 2)))
+        self.generator_model.add(LeakyReLU(alpha=self.alpha))
+        self.generator_model.add(Dropout(self.dropout))
+
+        self.generator_model.add(Dense(self.number_of_features, activation='sigmoid'))
+        optimizer = Adam(lr=self.learning_rate)
+        self.generator_model.compile(loss='categorical_crossentropy', optimizer=optimizer)
+
+        # self.generator_model.summary()
+
+    def train_gg(self, x_train, y_train, epochs, batch_size, model_name, data, output_path, to_plot=False):
+        """
+        This function running the training stage manually.
+        :param output_path: Path to save loss fig
+        :param to_plot: Plots the losses if True
+        :param x_train: the training set features
+        :param y_train: the training set classes
+        :param model_name: name of model to save (for generator)
+        :param epochs: number of epochs
+        :param batch_size: the batch size
+        :return: trains the discriminator and generator.
+        """
+
+        losses_path = os.path.join(self.saved_models_path, f'{model_name}_losses')
+        model_file = os.path.join(self.saved_models_path, f'{model_name}_part_2_gen_weights.h5')
+        
+        # First train the discriminator
+        self.train_black_box_dis(x_train, y_train)
+        self.train_generator(x_train, model_file, epochs, batch_size, losses_path)
+        if to_plot:
+            self.plot_losses(data, output_path)
+
+    def train_black_box_dis(self, x_train, y_train):
+        """
+        Trains the discriminator and saves it.
+        :param x_train: the training set features
+        :param y_train: the training set classes
+        :return:
+        """
+        dis_output = os.path.join(self.saved_models_path, 'black_box_dis_model')
+
+        if os.path.exists(dis_output):
+            # print('Blackbox discriminator already trained')
+            with open(dis_output, 'rb') as rf_file:
+                self.discriminator_model = pickle.load(rf_file)
+
+        self.discriminator_model.fit(x_train, y_train)
+        with open(dis_output, 'wb') as rf_file:
+            pickle.dump(self.discriminator_model, rf_file)
+
+    def train_generator(self, data, model_path, epochs, start_batch_size, losses_path):
+        """
+        Function for training the general generator.
+        :param losses_path: The filepath for the loss results
+        :param data: The normalized dataset
+        :param model_path: The name of the model to save. includes epoch size, batches etc.
+        :param epochs: Number of epochs
+        :param start_batch_size: Size of batch to use.
+        :return: trains the generator, saves it and the losses during training.
+        """
+
+        if os.path.exists(model_path):
+            self.generator_model.load_weights(model_path)
+            with open(losses_path, 'rb') as loss_file:
+                self.losses = pickle.load(loss_file)
+            return
+
+        for epoch in range(epochs):  # iterates over the epochs
+            np.random.shuffle(data)
+            batch_size = start_batch_size
+            for i in tqdm(range(0, data.shape[0], batch_size), ascii=True):  # Iterate over batches
+                if data.shape[0] - i >= batch_size:
+                    batch_input = data[i:i + batch_size]
+                else:  # The last iteration
+                    batch_input = data[i:]
+                    batch_size = batch_input.shape[0]
+
+                g_loss = self.train_generator_on_batch(batch_input)
+                self.losses['gen_loss'].append(g_loss)
+
+        self.save_generator_model(model_path, losses_path)
+
+    def save_generator_model(self, generator_model_path, losses_path):
+        """
+        Saves the model and the loss data with pickle.
+
+        :param generator_model_path: File path for the generator
+        :param losses_path: File path for the losses
+        :return:
+        """
+        self.generator_model.save_weights(generator_model_path)
+        with open(losses_path, 'wb+') as loss_file:
+            pickle.dump(self.losses, loss_file)
+
+    def train_generator_on_batch(self, batch_input):
+        """
+        Trains the generator for a single batch. Creates the necessary input, comprised of noise and the real
+        probabilities obtained from the black box. Compared to the target output, made of real samples and the
+        probabilities made up by the generator.
+        :param batch_input:
+        :return:
+        """
+        batch_size = batch_input.shape[0]
+        discriminator_probabilities = self.discriminator_model.predict_proba(batch_input)[:, -1:]
+        # noise = randn(self.noise_dim * batch_size).reshape((batch_size, self.noise_dim))
+
+        noise = randn(batch_size, self.noise_dim)
+        gen_model_input = np.hstack([noise, discriminator_probabilities])
+        generated_probabilities = self.generator_model.predict(gen_model_input)[:, -1:]  # Take only probabilities
+        target_output = np.hstack([batch_input, generated_probabilities])
+        g_loss = self.generator_model.train_on_batch(gen_model_input, target_output)  # The actual training
+
+        return g_loss
+
+    def plot_discriminator_results(self, x_test, y_test, data, path):
+        """
+        :param x_test: Test set
+        :param y_test: Test classes
+        :return: Prints the required plots.
+        """
+
+        blackbox_probs = self.discriminator_model.predict_proba(x_test)
+        discriminator_predictions = self.discriminator_model.predict(x_test)
+        count_1 = int(np.sum(y_test))
+        count_0 = int(y_test.shape[0] - count_1)
+        class_data = (['Class 0', 'Class 1'], [count_0, count_1])
+        self.plot_data(class_data, path, mode='bar', x_title='Class', title=f'Distribution of classes - {data} dataset')
+        self.plot_data(blackbox_probs[:, 0], path, title=f'Probabilities for test set - class 0 - {data} dataset')
+        self.plot_data(blackbox_probs[:, 1], path, title=f'Probabilities for test set - class 1 - {data} dataset')
+
+        min_confidence = blackbox_probs[:, 0].min(), blackbox_probs[:, 1].min()
+        max_confidence = blackbox_probs[:, 0].max(), blackbox_probs[:, 1].max()
+        mean_confidence = blackbox_probs[:, 0].mean(), blackbox_probs[:, 1].mean()
+
+        print("Accuracy:", metrics.accuracy_score(y_test, discriminator_predictions))
+        for c in [0, 1]:
+            print(f'Class {c} - Min confidence: {min_confidence[c]} - Max Confidence: {max_confidence[c]} - '
+                  f'Mean confidence: {mean_confidence[c]}')
+
+    def plot_generator_results(self, data, path, num_of_instances=1000):
+        """
+        Creates plots for the generator results on 1000 instances.
+        :param path:
+        :param data: Name of dataset used.
+        :param num_of_instances: Number of samples to generate.
+        :return:
+        """
+        sampled_proba, generated_instances = self.generate_n_samples(num_of_instances)
+
+        proba_fake = self.discriminator_model.predict_proba(generated_instances[:, :-1])
+        for c in [0, 1]:
+            title = f'Confidence Score for Class {c} of Fake Samples - {data} dataset'
+            self.plot_data(proba_fake[:, c], path, x_title='Confidence Score', title=title)
+
+        black_box_confidence = proba_fake[:, 1:]
+        proba_error = np.abs(sampled_proba - black_box_confidence)
+        generated_classes = np.array([int(round(c)) for c in generated_instances[:, -1].tolist()]).reshape(1000, 1)
+        proba_stats = np.hstack([sampled_proba, generated_classes, proba_fake[:, :1], proba_fake[:, 1:], proba_error])
+
+        for c in [0, 1]:
+            class_data = proba_stats[proba_stats[:, 1] == c]
+            class_data = class_data[class_data[:, 0].argsort()]  # Sort it for the plot
+            title = f'Error rate for different probabilities, class {c} - {data} dataset'
+            self.plot_data((class_data[:, 0], class_data[:, -1]), path, mode='plot', y_title='error rate', title=title)
+
+    def generate_n_samples(self, n):
+        """
+        Functions for generating N samples with a uniformly distribution confidence level.
+        :param n: Number of samples
+        :return: a tuple of the confidence scores used and the samples created.
+        """
+        noise = randn(n, self.noise_dim)
+        # confidences = np.sort(np.random.uniform(0, 1, (n, 1)), axis=0)
+        confidences = np.random.uniform(0, 1, (n, 1))
+
+        generator_input = np.hstack([noise, confidences])  # Stick them together
+        generated_instances = self.generator_model.predict(generator_input)  # Create samples
+
+        return confidences, generated_instances
+
+    @staticmethod
+    def plot_data(data, path, mode='hist', x_title='Probabilities', y_title='# of Instances', title='Distribution'):
+        """
+        :param path: Path to save
+        :param mode: Mode to use
+        :param y_title: Title of y axis
+        :param x_title: Title of x axis
+        :param data: Data to plot
+        :param title: Title of plot
+        :return: Prints a plot
+        """
+        plt.clf()
+
+        if mode == 'hist':
+            plt.hist(data)
+        elif mode == 'bar':
+            plt.bar(data[0], data[1])
+        else:
+            plt.plot(data[0], data[1])
+
+        plt.title(title)
+        plt.ylabel(y_title)
+        plt.xlabel(x_title)
+        # plt.show()
+        path = os.path.join(path, title)
+        plt.savefig(path)
+
+    def plot_losses(self, data, path):
+        """
+        Plot the losses while training
+        :return:
+        """
+        plt.clf()
+        plt.plot(self.losses['gen_loss'])
+        plt.title('Model loss')
+        plt.ylabel('Loss')
+        plt.xlabel('Iteration')
+        # plt.show()
+        plt.savefig(os.path.join(path, f'{data} dataset - general_generator_loss.png'))
+
+    def get_error(self, num_of_instances=1000):
+        """
+        Calculates the error of the generator we created by measuring the difference between the probability that
+        was given as input and the probability of the discriminator on the sample created.
+        :param num_of_instances: Number of samples to generate.
+        :return: An array of errors.
+        """
+        sampled_proba, generated_instances = self.generate_n_samples(num_of_instances)
+        proba_fake = self.discriminator_model.predict_proba(generated_instances[:, :-1])
+        black_box_confidence = proba_fake[:, 1:]
+        return np.abs(sampled_proba - black_box_confidence)
+

outputs/empty

@@ -0,0 +1 @@
+