{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S1TmcWKsFaJV",
        "outputId": "62f83c4f-e1ec-4d1a-b42d-48c8b36c4634"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting confluent_kafka\n",
            "  Downloading confluent_kafka-2.14.0-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (32 kB)\n",
            "Downloading confluent_kafka-2.14.0-cp312-cp312-manylinux_2_28_x86_64.whl (4.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.0/4.0 MB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: confluent_kafka\n",
            "Successfully installed confluent_kafka-2.14.0\n"
          ]
        }
      ],
      "source": [
        "pip install confluent_kafka"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "qkclaz2CFIUx",
        "outputId": "90f024c7-0833-4618-fb11-22d706209aa9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Consumer with ML started...\n",
            "NORMAL: {'user_id': 95, 'amount': 255.75, 'timestamp': 1778510472.8820539}\n",
            "NORMAL: {'user_id': 66, 'amount': 188.55, 'timestamp': 1778510474.9414551}\n",
            "NORMAL: {'user_id': 97, 'amount': 485.02, 'timestamp': 1778510479.0595143}\n",
            "NORMAL: {'user_id': 40, 'amount': 191.78, 'timestamp': 1778510480.0892992}\n",
            "NORMAL: {'user_id': 45, 'amount': 54.1, 'timestamp': 1778510481.1198835}\n",
            "NORMAL: {'user_id': 84, 'amount': 170.68, 'timestamp': 1778510484.2095373}\n",
            "NORMAL: {'user_id': 40, 'amount': 117.62, 'timestamp': 1778510485.239053}\n",
            "NORMAL: {'user_id': 39, 'amount': 131.73, 'timestamp': 1778510492.4461105}\n",
            "ML ANOMALY: {'user_id': 42, 'amount': 4334.36, 'timestamp': 1778510493.4759965}\n",
            "NORMAL: {'user_id': 74, 'amount': 213.47, 'timestamp': 1778510495.5363402}\n",
            "NORMAL: {'user_id': 92, 'amount': 378.84, 'timestamp': 1778510496.5715802}\n",
            "NORMAL: {'user_id': 28, 'amount': 124.56, 'timestamp': 1778510497.6012444}\n",
            "NORMAL: {'user_id': 95, 'amount': 260.0, 'timestamp': 1778510498.6312954}\n",
            "NORMAL: {'user_id': 73, 'amount': 342.64, 'timestamp': 1778510499.661054}\n",
            "NORMAL: {'user_id': 16, 'amount': 147.11, 'timestamp': 1778510500.6909337}\n",
            "NORMAL: {'user_id': 22, 'amount': 486.78, 'timestamp': 1778510502.7514715}\n",
            "NORMAL: {'user_id': 32, 'amount': 73.19, 'timestamp': 1778510504.811493}\n",
            "NORMAL: {'user_id': 19, 'amount': 188.92, 'timestamp': 1778510510.9892082}\n",
            "NORMAL: {'user_id': 71, 'amount': 218.79, 'timestamp': 1778510512.0188704}\n",
            "NORMAL: {'user_id': 75, 'amount': 336.77, 'timestamp': 1778510513.0480578}\n",
            "NORMAL: {'user_id': 82, 'amount': 251.2, 'timestamp': 1778510514.077877}\n",
            "NORMAL: {'user_id': 2, 'amount': 396.15, 'timestamp': 1778510515.1091313}\n",
            "NORMAL: {'user_id': 57, 'amount': 315.5, 'timestamp': 1778510516.1391258}\n",
            "NORMAL: {'user_id': 80, 'amount': 485.19, 'timestamp': 1778510524.451607}\n",
            "NORMAL: {'user_id': 63, 'amount': 185.21, 'timestamp': 1778510525.4813645}\n",
            "ML ANOMALY: {'user_id': 100, 'amount': 11.99, 'timestamp': 1778510526.5112405}\n",
            "NORMAL: {'user_id': 73, 'amount': 198.44, 'timestamp': 1778510527.5414555}\n",
            "ML ANOMALY: {'user_id': 28, 'amount': 4196.47, 'timestamp': 1778510530.6312032}\n",
            "NORMAL: {'user_id': 20, 'amount': 213.65, 'timestamp': 1778510534.749918}\n",
            "NORMAL: {'user_id': 55, 'amount': 196.65, 'timestamp': 1778510535.7800546}\n",
            "NORMAL: {'user_id': 49, 'amount': 410.1, 'timestamp': 1778510536.810472}\n",
            "NORMAL: {'user_id': 26, 'amount': 245.71, 'timestamp': 1778510539.9006226}\n",
            "NORMAL: {'user_id': 92, 'amount': 475.35, 'timestamp': 1778510540.9302115}\n",
            "NORMAL: {'user_id': 19, 'amount': 34.94, 'timestamp': 1778510541.960051}\n",
            "NORMAL: {'user_id': 57, 'amount': 432.0, 'timestamp': 1778510544.019122}\n",
            "NORMAL: {'user_id': 84, 'amount': 481.49, 'timestamp': 1778510545.0491996}\n",
            "NORMAL: {'user_id': 60, 'amount': 85.26, 'timestamp': 1778510546.0786042}\n",
            "NORMAL: {'user_id': 14, 'amount': 304.38, 'timestamp': 1778510547.1083302}\n",
            "NORMAL: {'user_id': 70, 'amount': 147.74, 'timestamp': 1778510549.1679802}\n",
            "NORMAL: {'user_id': 60, 'amount': 333.41, 'timestamp': 1778510550.1975765}\n",
            "NORMAL: {'user_id': 48, 'amount': 433.15, 'timestamp': 1778510551.2281058}\n",
            "NORMAL: {'user_id': 41, 'amount': 85.29, 'timestamp': 1778510552.2577817}\n",
            "NORMAL: {'user_id': 7, 'amount': 162.84, 'timestamp': 1778510554.3175669}\n",
            "NORMAL: {'user_id': 83, 'amount': 464.04, 'timestamp': 1778510556.3778625}\n",
            "NORMAL: {'user_id': 73, 'amount': 411.22, 'timestamp': 1778510557.4082847}\n",
            "NORMAL: {'user_id': 70, 'amount': 115.06, 'timestamp': 1778510558.4381568}\n",
            "NORMAL: {'user_id': 32, 'amount': 374.36, 'timestamp': 1778510560.4979246}\n",
            "NORMAL: {'user_id': 87, 'amount': 461.33, 'timestamp': 1778510564.6191485}\n",
            "NORMAL: {'user_id': 74, 'amount': 300.94, 'timestamp': 1778510566.679175}\n",
            "NORMAL: {'user_id': 52, 'amount': 105.67, 'timestamp': 1778510567.735611}\n",
            "NORMAL: {'user_id': 26, 'amount': 417.42, 'timestamp': 1778510570.8244543}\n",
            "NORMAL: {'user_id': 70, 'amount': 243.62, 'timestamp': 1778510573.9132168}\n",
            "NORMAL: {'user_id': 80, 'amount': 354.25, 'timestamp': 1778510574.9424686}\n",
            "NORMAL: {'user_id': 89, 'amount': 240.9, 'timestamp': 1778510575.9721017}\n",
            "NORMAL: {'user_id': 45, 'amount': 394.76, 'timestamp': 1778510577.0015826}\n",
            "NORMAL: {'user_id': 97, 'amount': 130.98, 'timestamp': 1778510578.031023}\n",
            "NORMAL: {'user_id': 74, 'amount': 358.03, 'timestamp': 1778510580.090143}\n",
            "NORMAL: {'user_id': 7, 'amount': 318.58, 'timestamp': 1778510581.1205785}\n",
            "NORMAL: {'user_id': 77, 'amount': 222.7, 'timestamp': 1778510583.180435}\n",
            "NORMAL: {'user_id': 22, 'amount': 444.6, 'timestamp': 1778510584.2101583}\n",
            "NORMAL: {'user_id': 55, 'amount': 425.71, 'timestamp': 1778510585.240488}\n",
            "NORMAL: {'user_id': 32, 'amount': 439.39, 'timestamp': 1778510587.3012028}\n",
            "NORMAL: {'user_id': 22, 'amount': 275.68, 'timestamp': 1778510594.5107589}\n",
            "NORMAL: {'user_id': 2, 'amount': 196.58, 'timestamp': 1778510595.540974}\n",
            "NORMAL: {'user_id': 63, 'amount': 390.6, 'timestamp': 1778510596.5705566}\n",
            "ML ANOMALY: {'user_id': 70, 'amount': 4451.37, 'timestamp': 1778510600.6905274}\n",
            "NORMAL: {'user_id': 22, 'amount': 110.97, 'timestamp': 1778510601.7199452}\n",
            "NORMAL: {'user_id': 67, 'amount': 106.08, 'timestamp': 1778510602.7501926}\n",
            "NORMAL: {'user_id': 34, 'amount': 25.66, 'timestamp': 1778510603.7799377}\n",
            "NORMAL: {'user_id': 55, 'amount': 241.88, 'timestamp': 1778510606.870125}\n",
            "NORMAL: {'user_id': 52, 'amount': 281.47, 'timestamp': 1778510608.9302537}\n",
            "NORMAL: {'user_id': 42, 'amount': 233.45, 'timestamp': 1778510612.019618}\n",
            "NORMAL: {'user_id': 87, 'amount': 225.41, 'timestamp': 1778510614.0801058}\n",
            "NORMAL: {'user_id': 45, 'amount': 20.1, 'timestamp': 1778510615.1094768}\n",
            "NORMAL: {'user_id': 82, 'amount': 208.65, 'timestamp': 1778510616.1392539}\n",
            "NORMAL: {'user_id': 82, 'amount': 116.23, 'timestamp': 1778510619.2289767}\n",
            "NORMAL: {'user_id': 16, 'amount': 375.88, 'timestamp': 1778510620.2585032}\n",
            "NORMAL: {'user_id': 81, 'amount': 42.36, 'timestamp': 1778510621.2880607}\n",
            "NORMAL: {'user_id': 74, 'amount': 173.58, 'timestamp': 1778510623.3480804}\n",
            "NORMAL: {'user_id': 20, 'amount': 41.48, 'timestamp': 1778510624.3781543}\n",
            "NORMAL: {'user_id': 77, 'amount': 23.71, 'timestamp': 1778510626.4382935}\n",
            "NORMAL: {'user_id': 40, 'amount': 22.51, 'timestamp': 1778510627.4680707}\n",
            "NORMAL: {'user_id': 92, 'amount': 337.84, 'timestamp': 1778510628.5001843}\n",
            "NORMAL: {'user_id': 45, 'amount': 459.14, 'timestamp': 1778510630.5599825}\n",
            "NORMAL: {'user_id': 95, 'amount': 128.94, 'timestamp': 1778510633.6491964}\n",
            "NORMAL: {'user_id': 82, 'amount': 409.69, 'timestamp': 1778510634.6788418}\n",
            "NORMAL: {'user_id': 95, 'amount': 19.62, 'timestamp': 1778510639.828434}\n",
            "NORMAL: {'user_id': 19, 'amount': 373.67, 'timestamp': 1778510642.9178803}\n",
            "NORMAL: {'user_id': 19, 'amount': 285.78, 'timestamp': 1778510643.9475126}\n",
            "NORMAL: {'user_id': 40, 'amount': 469.96, 'timestamp': 1778510649.0979857}\n",
            "NORMAL: {'user_id': 40, 'amount': 74.58, 'timestamp': 1778510651.1581306}\n",
            "NORMAL: {'user_id': 40, 'amount': 150.55, 'timestamp': 1778510652.1881502}\n",
            "NORMAL: {'user_id': 42, 'amount': 400.86, 'timestamp': 1778510655.2782469}\n",
            "NORMAL: {'user_id': 40, 'amount': 79.78, 'timestamp': 1778510657.3380325}\n",
            "NORMAL: {'user_id': 93, 'amount': 214.69, 'timestamp': 1778510658.3684695}\n",
            "NORMAL: {'user_id': 16, 'amount': 390.8, 'timestamp': 1778510659.3979876}\n",
            "NORMAL: {'user_id': 89, 'amount': 428.25, 'timestamp': 1778510661.4570332}\n",
            "NORMAL: {'user_id': 92, 'amount': 340.55, 'timestamp': 1778510662.4864717}\n",
            "NORMAL: {'user_id': 94, 'amount': 291.68, 'timestamp': 1778510663.5158875}\n",
            "NORMAL: {'user_id': 2, 'amount': 334.25, 'timestamp': 1778510664.5462298}\n",
            "ML ANOMALY: {'user_id': 19, 'amount': 12.61, 'timestamp': 1778510665.576023}\n",
            "NORMAL: {'user_id': 20, 'amount': 382.52, 'timestamp': 1778510666.6054597}\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m/tmp/ipykernel_6399/1565764707.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m     \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconsumer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmsg\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "import json\n",
        "import numpy as np\n",
        "from confluent_kafka import Consumer\n",
        "from sklearn.ensemble import IsolationForest\n",
        "\n",
        "conf = {\n",
        "    'bootstrap.servers': 'pkc-oxqxx9.us-east-1.aws.confluent.cloud:9092',\n",
        "    'security.protocol': 'SASL_SSL',\n",
        "    'sasl.mechanisms': 'PLAIN',\n",
        "    'sasl.username': 'API_KEY',\n",
        "    'sasl.password': 'PASSWORD',\n",
        "    'group.id': 'stream-group',\n",
        "    'auto.offset.reset': 'latest'\n",
        "}\n",
        "\n",
        "consumer = Consumer(conf)\n",
        "consumer.subscribe(['stream_demo'])\n",
        "\n",
        "print(\"Consumer with ML started...\")\n",
        "\n",
        "window = []\n",
        "model = IsolationForest(contamination=0.05)\n",
        "\n",
        "while True:\n",
        "    msg = consumer.poll(1.0)\n",
        "\n",
        "    if msg is None:\n",
        "        continue\n",
        "    if msg.error():\n",
        "        print(\"Error:\", msg.error())\n",
        "        continue\n",
        "\n",
        "    data = json.loads(msg.value().decode('utf-8'))\n",
        "    amount = data[\"amount\"]\n",
        "\n",
        "    # collect data\n",
        "    window.append([amount])\n",
        "\n",
        "    # train after enough data\n",
        "    if len(window) > 20:\n",
        "        model.fit(window)\n",
        "\n",
        "        pred = model.predict([[amount]])\n",
        "\n",
        "        if pred[0] == -1:\n",
        "            print(\"ML ANOMALY:\", data)\n",
        "        else:\n",
        "            print(\"NORMAL:\", data)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}