From d9b8fcf98754d2433bd72ea31c42ab2886116a06 Mon Sep 17 00:00:00 2001 From: CoprDistGit Date: Mon, 29 May 2023 13:15:16 +0000 Subject: automatic import of python-sparkmonitor --- .gitignore | 1 + python-sparkmonitor.spec | 265 +++++++++++++++++++++++++++++++++++++++++++++++ sources | 1 + 3 files changed, 267 insertions(+) create mode 100644 python-sparkmonitor.spec create mode 100644 sources diff --git a/.gitignore b/.gitignore index e69de29..fb36ab3 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +/sparkmonitor-2.1.1.tar.gz diff --git a/python-sparkmonitor.spec b/python-sparkmonitor.spec new file mode 100644 index 0000000..f7af871 --- /dev/null +++ b/python-sparkmonitor.spec @@ -0,0 +1,265 @@ +%global _empty_manifest_terminate_build 0 +Name: python-sparkmonitor +Version: 2.1.1 +Release: 1 +Summary: Jupyter Notebook & Lab extension to monitor Apache Spark jobs from a notebook +License: Apache-2.0 +URL: https://github.com/swan-cern/sparkmonitor#readme +Source0: https://mirrors.nju.edu.cn/pypi/web/packages/2f/33/df61033427bc0d110cf489cb703e23b78803d4f6ac7ef4000ef59f7300dc/sparkmonitor-2.1.1.tar.gz +BuildArch: noarch + +Requires: python3-jupyterlab + +%description +![jobdisplay](https://user-images.githubusercontent.com/6822941/29753710-ff8849b6-8b94-11e7-8f9c-bdc59bf72143.gif) +## Requirements +- Jupyter Lab 3 OR Jupyter Notebook 4.4.0 or higher +- pyspark 2 or 3 +## Features +- Automatically displays a live monitoring tool below cells that run Spark jobs in a Jupyter notebook +- A table of jobs and stages with progressbars +- A timeline which shows jobs, stages, and tasks +- A graph showing number of active tasks & executor cores vs time + + + + + + +
+## Quick Start +### Setting up the extension +```bash +pip install sparkmonitor # install the extension +# set up an ipython profile and add our kernel extension to it +ipython profile create # if it does not exist +echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >> $(ipython profile locate default)/ipython_kernel_config.py +# For use with jupyter notebook install and enable the nbextension +jupyter nbextension install sparkmonitor --py +jupyter nbextension enable sparkmonitor --py +# The jupyterlab extension is automatically enabled +``` +With the extension installed, a `SparkConf` object called `conf` will be usable from your notebooks. You can use it as follows: +```python +from pyspark import SparkContext +# Start the spark context using the SparkConf object named `conf` the extension created in your kernel. +sc=SparkContext.getOrCreate(conf=conf) +``` +If you already have your own spark configuration, you will need to set `spark.extraListeners` to `sparkmonitor.listener.JupyterSparkMonitorListener` and `spark.driver.extraClassPath` to the path to the sparkmonitor python package `path/to/package/sparkmonitor/listener_.jar` +```python +from pyspark.sql import SparkSession +spark = SparkSession.builder\ + .config('spark.extraListeners', 'sparkmonitor.listener.JupyterSparkMonitorListener')\ + .config('spark.driver.extraClassPath', 'venv/lib/python3./site-packages/sparkmonitor/listener_.jar')\ + .getOrCreate() +``` +## Development +If you'd like to develop the extension: +```bash +# See package.json scripts for building the frontend +yarn run build: +# Install the package in editable mode +pip install -e . +# Symlink jupyterlab extension +jupyter labextension develop --overwrite . +# Watch for frontend changes +yarn run watch +# Build the spark JAR files +sbt +package +``` +## History +- This project was originally written by krishnan-r as a [Google Summer of Code project](https://github.com/krishnan-r/sparkmonitor) for Jupyter Notebook with the [SWAN](https://swan.web.cern.ch/swan/) Notebook Service team at [CERN](http://home.cern/). +- Further fixes and improvements were made by the team at CERN and members of the community maintained at [swan-cern/jupyter-extensions/tree/master/SparkMonitor](https://github.com/swan-cern/jupyter-extensions/tree/master/SparkMonitor) +- [Jafer Haider](https://github.com/itsjafer) created the fork [jupyterlab-sparkmonitor](https://github.com/itsjafer/jupyterlab-sparkmonitor) to update the extension to be compatible with JupyterLab as part of his internship at Yelp. +- This repository merges all the work done above and provides support for Lab & Notebook from a single package. +## Changelog +This repository is published to pypi as [sparkmonitor](https://pypi.org/project/sparkmonitor/) +- 2.x see the [github releases page](https://github.com/swan-cern/sparkmonitor/releases) of this repository +- 1.x and below were published from [swan-cern/jupyter-extensions](https://github.com/swan-cern/jupyter-extensions) and some initial versions from [krishnan-r/sparkmonitor](https://github.com/krishnan-r/sparkmonitor) + +%package -n python3-sparkmonitor +Summary: Jupyter Notebook & Lab extension to monitor Apache Spark jobs from a notebook +Provides: python-sparkmonitor +BuildRequires: python3-devel +BuildRequires: python3-setuptools +BuildRequires: python3-pip +%description -n python3-sparkmonitor +![jobdisplay](https://user-images.githubusercontent.com/6822941/29753710-ff8849b6-8b94-11e7-8f9c-bdc59bf72143.gif) +## Requirements +- Jupyter Lab 3 OR Jupyter Notebook 4.4.0 or higher +- pyspark 2 or 3 +## Features +- Automatically displays a live monitoring tool below cells that run Spark jobs in a Jupyter notebook +- A table of jobs and stages with progressbars +- A timeline which shows jobs, stages, and tasks +- A graph showing number of active tasks & executor cores vs time + + + + + + +
+## Quick Start +### Setting up the extension +```bash +pip install sparkmonitor # install the extension +# set up an ipython profile and add our kernel extension to it +ipython profile create # if it does not exist +echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >> $(ipython profile locate default)/ipython_kernel_config.py +# For use with jupyter notebook install and enable the nbextension +jupyter nbextension install sparkmonitor --py +jupyter nbextension enable sparkmonitor --py +# The jupyterlab extension is automatically enabled +``` +With the extension installed, a `SparkConf` object called `conf` will be usable from your notebooks. You can use it as follows: +```python +from pyspark import SparkContext +# Start the spark context using the SparkConf object named `conf` the extension created in your kernel. +sc=SparkContext.getOrCreate(conf=conf) +``` +If you already have your own spark configuration, you will need to set `spark.extraListeners` to `sparkmonitor.listener.JupyterSparkMonitorListener` and `spark.driver.extraClassPath` to the path to the sparkmonitor python package `path/to/package/sparkmonitor/listener_.jar` +```python +from pyspark.sql import SparkSession +spark = SparkSession.builder\ + .config('spark.extraListeners', 'sparkmonitor.listener.JupyterSparkMonitorListener')\ + .config('spark.driver.extraClassPath', 'venv/lib/python3./site-packages/sparkmonitor/listener_.jar')\ + .getOrCreate() +``` +## Development +If you'd like to develop the extension: +```bash +# See package.json scripts for building the frontend +yarn run build: +# Install the package in editable mode +pip install -e . +# Symlink jupyterlab extension +jupyter labextension develop --overwrite . +# Watch for frontend changes +yarn run watch +# Build the spark JAR files +sbt +package +``` +## History +- This project was originally written by krishnan-r as a [Google Summer of Code project](https://github.com/krishnan-r/sparkmonitor) for Jupyter Notebook with the [SWAN](https://swan.web.cern.ch/swan/) Notebook Service team at [CERN](http://home.cern/). +- Further fixes and improvements were made by the team at CERN and members of the community maintained at [swan-cern/jupyter-extensions/tree/master/SparkMonitor](https://github.com/swan-cern/jupyter-extensions/tree/master/SparkMonitor) +- [Jafer Haider](https://github.com/itsjafer) created the fork [jupyterlab-sparkmonitor](https://github.com/itsjafer/jupyterlab-sparkmonitor) to update the extension to be compatible with JupyterLab as part of his internship at Yelp. +- This repository merges all the work done above and provides support for Lab & Notebook from a single package. +## Changelog +This repository is published to pypi as [sparkmonitor](https://pypi.org/project/sparkmonitor/) +- 2.x see the [github releases page](https://github.com/swan-cern/sparkmonitor/releases) of this repository +- 1.x and below were published from [swan-cern/jupyter-extensions](https://github.com/swan-cern/jupyter-extensions) and some initial versions from [krishnan-r/sparkmonitor](https://github.com/krishnan-r/sparkmonitor) + +%package help +Summary: Development documents and examples for sparkmonitor +Provides: python3-sparkmonitor-doc +%description help +![jobdisplay](https://user-images.githubusercontent.com/6822941/29753710-ff8849b6-8b94-11e7-8f9c-bdc59bf72143.gif) +## Requirements +- Jupyter Lab 3 OR Jupyter Notebook 4.4.0 or higher +- pyspark 2 or 3 +## Features +- Automatically displays a live monitoring tool below cells that run Spark jobs in a Jupyter notebook +- A table of jobs and stages with progressbars +- A timeline which shows jobs, stages, and tasks +- A graph showing number of active tasks & executor cores vs time + + + + + + +
+## Quick Start +### Setting up the extension +```bash +pip install sparkmonitor # install the extension +# set up an ipython profile and add our kernel extension to it +ipython profile create # if it does not exist +echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >> $(ipython profile locate default)/ipython_kernel_config.py +# For use with jupyter notebook install and enable the nbextension +jupyter nbextension install sparkmonitor --py +jupyter nbextension enable sparkmonitor --py +# The jupyterlab extension is automatically enabled +``` +With the extension installed, a `SparkConf` object called `conf` will be usable from your notebooks. You can use it as follows: +```python +from pyspark import SparkContext +# Start the spark context using the SparkConf object named `conf` the extension created in your kernel. +sc=SparkContext.getOrCreate(conf=conf) +``` +If you already have your own spark configuration, you will need to set `spark.extraListeners` to `sparkmonitor.listener.JupyterSparkMonitorListener` and `spark.driver.extraClassPath` to the path to the sparkmonitor python package `path/to/package/sparkmonitor/listener_.jar` +```python +from pyspark.sql import SparkSession +spark = SparkSession.builder\ + .config('spark.extraListeners', 'sparkmonitor.listener.JupyterSparkMonitorListener')\ + .config('spark.driver.extraClassPath', 'venv/lib/python3./site-packages/sparkmonitor/listener_.jar')\ + .getOrCreate() +``` +## Development +If you'd like to develop the extension: +```bash +# See package.json scripts for building the frontend +yarn run build: +# Install the package in editable mode +pip install -e . +# Symlink jupyterlab extension +jupyter labextension develop --overwrite . +# Watch for frontend changes +yarn run watch +# Build the spark JAR files +sbt +package +``` +## History +- This project was originally written by krishnan-r as a [Google Summer of Code project](https://github.com/krishnan-r/sparkmonitor) for Jupyter Notebook with the [SWAN](https://swan.web.cern.ch/swan/) Notebook Service team at [CERN](http://home.cern/). +- Further fixes and improvements were made by the team at CERN and members of the community maintained at [swan-cern/jupyter-extensions/tree/master/SparkMonitor](https://github.com/swan-cern/jupyter-extensions/tree/master/SparkMonitor) +- [Jafer Haider](https://github.com/itsjafer) created the fork [jupyterlab-sparkmonitor](https://github.com/itsjafer/jupyterlab-sparkmonitor) to update the extension to be compatible with JupyterLab as part of his internship at Yelp. +- This repository merges all the work done above and provides support for Lab & Notebook from a single package. +## Changelog +This repository is published to pypi as [sparkmonitor](https://pypi.org/project/sparkmonitor/) +- 2.x see the [github releases page](https://github.com/swan-cern/sparkmonitor/releases) of this repository +- 1.x and below were published from [swan-cern/jupyter-extensions](https://github.com/swan-cern/jupyter-extensions) and some initial versions from [krishnan-r/sparkmonitor](https://github.com/krishnan-r/sparkmonitor) + +%prep +%autosetup -n sparkmonitor-2.1.1 + +%build +%py3_build + +%install +%py3_install +install -d -m755 %{buildroot}/%{_pkgdocdir} +if [ -d doc ]; then cp -arf doc %{buildroot}/%{_pkgdocdir}; fi +if [ -d docs ]; then cp -arf docs %{buildroot}/%{_pkgdocdir}; fi +if [ -d example ]; then cp -arf example %{buildroot}/%{_pkgdocdir}; fi +if [ -d examples ]; then cp -arf examples %{buildroot}/%{_pkgdocdir}; fi +pushd %{buildroot} +if [ -d usr/lib ]; then + find usr/lib -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/lib64 ]; then + find usr/lib64 -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/bin ]; then + find usr/bin -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/sbin ]; then + find usr/sbin -type f -printf "/%h/%f\n" >> filelist.lst +fi +touch doclist.lst +if [ -d usr/share/man ]; then + find usr/share/man -type f -printf "/%h/%f.gz\n" >> doclist.lst +fi +popd +mv %{buildroot}/filelist.lst . +mv %{buildroot}/doclist.lst . + +%files -n python3-sparkmonitor -f filelist.lst +%dir %{python3_sitelib}/* + +%files help -f doclist.lst +%{_docdir}/* + +%changelog +* Mon May 29 2023 Python_Bot - 2.1.1-1 +- Package Spec generated diff --git a/sources b/sources new file mode 100644 index 0000000..9225e89 --- /dev/null +++ b/sources @@ -0,0 +1 @@ +f51dce84f964145042e141e167f03e89 sparkmonitor-2.1.1.tar.gz -- cgit v1.2.3