@inproceedings{Frankle19:LTH,
  title={The Lottery Ticket Hypothesis: Finding Sparse, Trainable Neural Networks},
  author={Jonathan Frankle and Michael Carbin},
  booktitle={International Conference on Learning Representations},
  year={2019},
  url={https://openreview.net/forum?id=rJl-b3RcF7},
}



@inproceedings{Malach20:ProvingLTH_PruningIsAll,
  title={Proving the lottery ticket hypothesis: pruning is all you need},
  author={Malach, Eran and Yehudai, Gilad and Shalev-shwartz, Shai and Shamir, Ohad},
  booktitle={Proceedings of the 37th International Conference on Machine Learning},
  pages={6682--6691},
  year={2020}
}

@InProceedings{Li19:POPSpeedAccNAS,
author = {Li, Xin and Zhou, Yiming and Pan, Zheng and Feng, Jiashi},
title = {Partial Order Pruning: For Best Speed/Accuracy Trade-Off in Neural Architecture Search},
booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
month = {June},
year = {2019}
}

@article{Ding22:NAP,
title = {NAP: Neural architecture search with pruning},
journal = {Neurocomputing},
volume = {477},
pages = {85-95},
year = {2022},
issn = {0925-2312},
doi = {https://doi.org/10.1016/j.neucom.2021.12.002},
url = {https://www.sciencedirect.com/science/article/pii/S0925231221018361},
author = {Yadong Ding and Yu Wu and Chengyue Huang and Siliang Tang and Fei Wu and Yi Yang and Wenwu Zhu and Yueting Zhuang},
keywords = {Neural architecture search, Network pruning, Computer vision},
abstract = {There has been continuously increasing attention attracted by Neural Architecture Search (NAS). Due to its computational efficiency, gradient-based NAS methods like DARTS have become the most popular framework for NAS tasks. Nevertheless, as the search iterates, the derived model in previous NAS frameworks becomes dominated by skip-connects, causing the performance downfall. In this work, we present a novel approach to alleviate this issue, named Neural Architecture search with Pruning (NAP). Unlike prior differentiable architecture search works, our approach draws the idea from network pruning. We first train an over-parameterized network, including all candidate operations. Then we propose a criterion to prune the network. Based on a newly designed relaxation of architecture representation, NAP can derive the most potent model by removing trivial and redundant edges from the whole network topology. Experiments show the effectiveness of our proposed approach. Specifically, the model searched by NAP achieves state-of-the-art performances (2.48% test error) on CIFAR-10. We transfer the model to ImageNet and obtains a 25.1% test error with only 5.0 M parameters, which is on par with modern NAS methods.}
}

@article{Elksen19:NASSurvey,
  author  = {Thomas Elsken and Jan Hendrik Metzen and Frank Hutter},
  title   = {Neural Architecture Search: A Survey},
  journal = {Journal of Machine Learning Research},
  year    = {2019},
  volume  = {20},
  number  = {55},
  pages   = {1--21},
  url     = {http://jmlr.org/papers/v20/18-598.html}
}

@article{Gong14:CompressingQuantization,
  author    = {Yunchao Gong and
               Liu Liu and
               Ming Yang and
               Lubomir D. Bourdev},
  title     = {Compressing Deep Convolutional Networks using Vector Quantization},
  journal   = {CoRR},
  volume    = {abs/1412.6115},
  year      = {2014},
  url       = {http://arxiv.org/abs/1412.6115},
  eprinttype = {arXiv},
  eprint    = {1412.6115},
  timestamp = {Fri, 06 May 2022 15:54:00 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/GongLYB14.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@INPROCEEDINGS{Ko19:MUP,
  author={Ko, Vinnie and Oehmcke, Stefan and Gieseke, Fabian},
  booktitle={2019 IEEE International Conference on Big Data (Big Data)}, 
  title={Magnitude and Uncertainty Pruning Criterion for Neural Networks}, 
  year={2019},
  pages={2317-2326},
  doi={10.1109/BigData47090.2019.9005692}
}

@article{Hoefler21:SparsityDL:PruningaGrowthEfficientInferenceTrainingNN,
  author  = {Torsten Hoefler and Dan Alistarh and Tal Ben-Nun and Nikoli Dryden and Alexandra Peste},
  title   = {Sparsity in Deep Learning: Pruning and growth for efficient inference and training in neural networks},
  journal = {Journal of Machine Learning Research},
  year    = {2021},
  volume  = {22},
  number  = {241},
  pages   = {1--124},
  url     = {http://jmlr.org/papers/v22/21-0366.html}
}

@INPROCEEDINGS{Yang16:DEECNNuEAP,
  author={Yang, Tien-Ju and Chen, Yu-Hsin and Sze, Vivienne},
  booktitle={2017 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 
  title={Designing Energy-Efficient Convolutional Neural Networks Using Energy-Aware Pruning}, 
  year={2017},
  pages={6071-6079},
  doi={10.1109/CVPR.2017.643}
}

@misc{Hu16:NTDDNPAtEDA,
  author = {Hengyuan Hu and Rui Peng and Yu-Wing Tai and Chi-Keung Tang},
  title = {Network Trimming: A Data-Driven Neuron Pruning Approach towards Efficient Deep Architectures},
  journal = {CoRR},
  volume = {abs/1607.03250},
  year = {2016},
  url = {http://arxiv.org/abs/1607.03250}
}

@inproceedings{Han15:LbWCfENN,
  author = {Han, Song and Pool, Jeff and Tran, John and Dally, William J.},
  title = {Learning both weights and connections for efficient neural networks},
  booktitle = {Proceedings of the 28th International Conference on Neural Information Processing Systems},
  year = {2015},
  pages = {1135--1143}
}

@inproceedings{LeCun89:OBD,
  title={Optimal brain damage},
  author={LeCun, Yann and Denker, John and Solla, Sara},
  booktitle={Advances in neural information processing systems},
  volume={2},
  year={1989}
}

@article{Ivakhnenko68group,
  title={The Group Method of Data Handling, a Rival of the Method of Stochastic Approximation},
  author={Ivakhnenko, Alexey Grigorevich},
  journal={Soviet Automatic Control},
  volume={13},
  number={3},
  pages={43--55},
  year={1968}
}

@article{Ivakhnenko71UnitPruning,
  author={Ivakhnenko, A. G.},
  journal={IEEE Transactions on Systems, Man, and Cybernetics}, 
  title={Polynomial Theory of Complex Systems}, 
  year={1971},
  volume={SMC-1},
  number={4},
  pages={364-378},
  doi={10.1109/TSMC.1971.4308320}
}

@inproceedings{Molchanov17:PCNNfREI,
  title={Pruning Convolutional Neural Networks for Resource Efficient Inference},
  author={Pavlo Molchanov and Stephen Tyree and Tero Karras and Timo Aila and Jan Kautz},
  booktitle={International Conference on Learning Representations},
  year={2017},
  url={https://openreview.net/forum?id=SJGCiw5gl}
}

@INPROCEEDINGS{Liu17:LECNtNSLIMMING,
  author={Liu, Zhuang and Li, Jianguo and Shen, Zhiqiang and Huang, Gao and Yan, Shoumeng and Zhang, Changshui},
  booktitle={2017 IEEE International Conference on Computer Vision (ICCV)}, 
  title={Learning Efficient Convolutional Networks through Network Slimming}, 
  year={2017},
  pages={2755-2763},
  doi={10.1109/ICCV.2017.298}
}

@inproceedings{Blalock20:StateNNPruning,
  author = {Blalock, Davis and Gonzalez Ortiz, Jose Javier and Frankle, Jonathan and Guttag, John},
  title = {What is the State of Neural Network Pruning?},
  booktitle = {Proceedings of Machine Learning and Systems},
  volume = {2},
  pages = {129--146},
  year = {2020}
}

@inproceedings{Renda20:ComparingRewFTiNNPLTH,
  title={Comparing Rewinding and Fine-tuning in Neural Network Pruning},
  author={Alex Renda and Jonathan Frankle and Michael Carbin},
  booktitle={International Conference on Learning Representations},
  year={2020},
  url={https://openreview.net/forum?id=S1gSj0NKvB}
}

@inproceedings{Rachwan22:WinningLTHAhead,
  title={Winning the lottery ahead of time: Efficient early network pruning},
  author={Rachwan, John et al.},
  booktitle={International Conference on Machine Learning},
  pages={18293--18309},
  year={2022}
}

@InProceedings{Widmann23:PruningforPowerOEEIoTNNP,
  author={Widmann, Thomas et al.},
  title={Pruning for Power: Optimizing Energy Efficiency in IoT with Neural Network Pruning},
  booktitle={Engineering Applications of Neural Networks},
  year={2023},
  pages={251--263}
}

@InProceedings{Kim22:ExploringLTHSNN,
  author="Kim, Youngeun et al.",
  title="Exploring Lottery Ticket Hypothesis in Spiking Neural Networks",
  booktitle="ECCV 2022",
  year="2022",
  pages="102--120"
}

@article{Liang21:PruningQuantizationDNNSurvey,
  title = {Pruning and quantization for deep neural network acceleration: A survey},
  journal = {Neurocomputing},
  volume = {461},
  pages = {370-403},
  year = {2021},
  author = {Tailin Liang et al.},
  doi = {10.1016/j.neucom.2021.07.045}
}

@article{Gholami22:SurveyQuantMethfENNInference,
  title={A survey of quantization methods for efficient neural network inference},
  author={Gholami, Amir et al.},
  booktitle={Low-Power Computer Vision},
  pages={291--326},
  year={2022}
}

@article{You19:DrawingEBTicketsEfficientTraining,
  author={You, Haoran et al.},
  title={Drawing early-bird tickets: Towards more efficient training of deep networks},
  journal={CoRR},
  volume={abs/1909.11957},
  year={2019},
  url={http://arxiv.org/abs/1909.11957}
}

@article{Hinton15:KD,
  author={Hinton, Geoffrey and Vinyals, Oriol and Dean, Jeff},
  title={Distilling the Knowledge in a Neural Network},
  journal={arXiv preprint arXiv:1503.02531},
  year={2015},
  url={https://arxiv.org/pdf/1503.02531}
}

@article{Cui21:JointSPruningandKD,
  title = {Joint structured pruning and dense knowledge distillation for efficient transformer model compression},
  journal = {Neurocomputing},
  volume = {458},
  pages = {56-69},
  year = {2021},
  author = {Cui, Baiyun et al.},
  doi = {10.1016/j.neucom.2021.05.084}
}

@inproceedings{Aghli21:CombiningPruningandKD,
  author = {Aghli, Nima and Ribeiro, Eraldo},
  title = {Combining Weight Pruning and Knowledge Distillation for CNN Compression},
  booktitle = {CVPR Workshops},
  year = {2021},
  pages = {3191--3198}
}

@inproceedings{HeZRS15:ResNet,
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  title={Deep Residual Learning for Image Recognition},
  booktitle={CVPR},
  year={2016},
  pages={770--778},
  doi={10.1109/CVPR.2016.90}
}

@misc{Xu15:EmpiricalEvalofReLUinCNNs,
  title={Empirical Evaluation of Rectified Activations in Convolutional Network},
  author={Bing Xu and Naiyan Wang and Tianqi Chen and Mu Li},
  year={2015},
  eprint={1505.00853},
  archivePrefix={arXiv}
}

@INPROCEEDINGS{Khalid20:EmpiricalEvaloActivationFinDCNNsforFacialExprR,
  author={Khalid, Muhammad et al.},
  title={Empirical Evaluation of Activation Functions in Deep Convolution Neural Network for Facial Expression Recognition},
  booktitle={2020 43rd International Conference on Telecommunications and Signal Processing (TSP)},
  year={2020},
  pages={204-207},
  doi={10.1109/TSP49548.2020.9163446}
}

@article{Yang24:LLMsSurvey,
  author={Yang, Jingfeng et al.},
  title={Harnessing the Power of LLMs in Practice: A Survey on ChatGPT and Beyond},
  journal={ACM Trans. Knowl. Discov. Data},
  year={2024},
  doi={10.1145/3649506}
}

@misc{Lee23:IPCC,
  author = {Lee, Hoesung et al.},
  title = {IPCC, 2023: Climate Change 2023: Synthesis Report},
  year = {2023},
  publisher = {Intergovernmental Panel on Climate Change (IPCC)},
  url = {https://www.ipcc.ch/report/ar6/syr/downloads/report/IPCC_AR6_SYR_LongerReport.pdf},
  doi = {10.59327/IPCC/AR6-9789291691647}
}

@article{Vadera22:MethodsPruningDNN,
  author={Vadera, Sunil and Ameen, Salem},
  journal={IEEE Access},
  title={Methods for Pruning Deep Neural Networks},
  year={2022},
  volume={10},
  pages={63280--63300},
  doi={10.1109/ACCESS.2022.3182659}
}

@article{Frankle19:StabilizingLTH,
       author = {{Frankle}, Jonathan and {Karolina Dziugaite}, Gintare and {Roy}, Daniel M. and {Carbin}, Michael},
        title = "{Stabilizing the Lottery Ticket Hypothesis}",
      journal = {arXiv e-prints},
     keywords = {Computer Science - Machine Learning, Computer Science - Computer Vision and Pattern Recognition, Statistics - Machine Learning},
         year = 2019,
        month = mar,
          eid = {arXiv:1903.01611  },
        pages = {arXiv:1903.01611  },
archivePrefix = {arXiv},
       eprint = {1903.01611},
 primaryClass = {cs.LG},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2019arXiv190301611F},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@inproceedings{Seuret19:DatasetEPBFontGroups,
    author = {Seuret, Mathias and Limbach, Saskia and Weichselbaumer, Nikolaus and Maier, Andreas and Christlein, Vincent},
    title = {Dataset of Pages from Early Printed Books with Multiple Font Groups},
    year = {2019},
    isbn = {9781450376686},
    publisher = {Association for Computing Machinery},
    address = {New York, NY, USA},    
    doi = {10.1145/3352631.3352640},
    abstract = {Based on contemporary scripts, early printers developed a large variety of different fonts. While fonts may slightly differ from one printer to another, they can be divided into font groups, such as Textura, Antiqua, or Fraktur. The recognition of font groups is important for computer scientists to select adequate OCR models, and of high interest to humanities scholars studying early printed books and the history of fonts. In this paper, we introduce a new, public dataset for the recognition of font groups in early printed books, and evaluate several state-of-the-art CNNs for the font group recognition task. The dataset consists of more than 35 600 page images, each page showing up to five different font groups, of which ten are considered in this dataset.},
    booktitle = {Proceedings of the 5th International Workshop on Historical Document Imaging and Processing},
    pages = {1–6},
    numpages = {6},
    keywords = {neural network, fonts, type, Fraktur, Gotico-Antiqua, Rotunda, Schwabacher, Greek, Hebrew, Textura, Antiqua, Italic, Bastarda, typography, dataset, book history, incunabula, digital humanities, historical documents, document analysis},
    location = {Sydney, NSW, Australia},
    series = {HIP '19}
}

@InProceedings{Seuret21:ICDARFontGroupDataClassification, 
    author="Seuret, Mathias
    and Nicolaou, Anguelos
    and Rodr{\'i}guez-Salas, Dalia
    and Weichselbaumer, Nikolaus
    and Stutzmann, Dominique
    and Mayr, Martin
    and Maier, Andreas
    and Christlein, Vincent",
    editor="Llad{\'o}s, Josep
    and Lopresti, Daniel
    and Uchida, Seiichi",
    title="ICDAR 2021 Competition on Historical Document Classification",
    booktitle="Document Analysis and Recognition -- ICDAR 2021",
    year="2021",
    publisher="Springer International Publishing",
    address="Cham",
    pages="618--634",
    abstract="This competition investigated the performance of historical document classification. The analysis of historical documents is a difficult challenge commonly solved by trained humanists. We provided three different classification tasks, which can be solved individually or jointly: font group/script type, location, date. The document images are provided by several institutions and are taken from handwritten and printed books as well as from charters. In contrast to previous competitions, all participants relied upon Deep Learning based approaches. Nevertheless, we saw a great performance variety of the different submitted systems. The easiest task seemed to be font group recognition while the script type classification and location classification were more challenging. In the dating task, the best system achieved a mean absolute error of about 22 years.",
    isbn="978-3-030-86337-1"
}

@misc{Kingma17:ADAM,
      title={Adam: A Method for Stochastic Optimization}, 
      author={Diederik P. Kingma and Jimmy Ba},
      year={2017},
      eprint={1412.6980},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{Nikolaidou22:SurveyHistoricalDocFontDataset,
  author={Konstantina Nikolaidou and Mathias Seuret and Hamam Mokayed and Marcus Liwicki},
  title={A survey of historical document image datasets},
  year={2022},
  cdate={1640995200000},
  journal={Int. J. Document Anal. Recognit.},
  volume={25},
  number={4},
  pages={305-338},
  url={https://doi.org/10.1007/s10032-022-00405-8},
}

@misc{Seuret19:ZenodoDatasetoPEPBwMFG,
  author       = {Seuret, Mathias and
                  Limbach, Saskia and
                  Weichselbaumer, Nikolaus and
                  Maier, Andreas and
                  Christlein, Vincent},
  title        = {{Dataset of Pages from Early Printed Books with 
                   Multiple Font Groups}},
  month        = aug,
  year         = 2019,
  publisher    = {Zenodo},
  version      = {1.0.0},
  doi          = {10.5281/zenodo.3366686 },
}

@misc{NVIDIA-SMI,
    author		= "{NVIDIA Corporation}",
    title			= "NVIDIA System Management Interface {nvidia-smi}",
    url="{https://developer.nvidia.com/nvidia-system-management-interface}",
    note = "Licensed by the NVIDIA-Corporation (08.01.2023)"
}

@misc{Schmidt21:CodeCarbon,
  title={CodeCarbon: Estimate and Track Carbon Emissions from Machine Learning Computing},
  author={Schmidt, Victor and Goyal, Kamal and Joshi, Aditya and Feld, Boris and Conell, Liam and Laskaris, Nikolas and Blank, Doug and Wilson, Jonathan and Friedler, Sorelle and Luccioni, Sasha},
  year={2021},
  doi={10.5281/zenodo.10594225},
}

@article{Gerum2020:SparsitytEvolPruning,
title = "Sparsity through evolutionary pruning prevents neuronal networks from overfitting",
journal = "Neural Networks",
volume = "128",
pages = "305 - 312",
year = "2020",
issn = "0893-6080",
doi = "https://doi.org/10.1016/j.neunet.2020.05.007",
url = "http://www.sciencedirect.com/science/article/pii/S0893608020301696",
author = "Richard C. Gerum and André Erpenbeck and Patrick Krauss and Achim Schilling",
keywords = "Evolution, Artificial neural networks, Maze task, Evolutionary algorithm, Overfitting, Biological plausibility",
abstract = "Modern Machine learning techniques take advantage of the exponentially rising calculation power in new generation processor units. Thus, the number of parameters which are trained to solve complex tasks was highly increased over the last decades. However, still the networks fail – in contrast to our brain – to develop general intelligence in the sense of being able to solve several complex tasks with only one network architecture. This could be the case because the brain is not a randomly initialized neural network, which has to be trained from scratch by simply investing a lot of calculation power, but has from birth some fixed hierarchical structure. To make progress in decoding the structural basis of biological neural networks we here chose a bottom-up approach, where we evolutionarily trained small neural networks in performing a maze task. This simple maze task requires dynamic decision making with delayed rewards. We were able to show that during the evolutionary optimization random severance of connections leads to better generalization performance of the networks compared to fully connected networks. We conclude that sparsity is a central property of neural networks and should be considered for modern Machine learning approaches."
}


@article{SeulKi21:TowardsCompactDNNvEAPruning,
  publtype={informal},
  author={Seul-Ki Yeom and Kyung-Hwan Shim and Jee-Hyun Hwang},
  title={Toward Compact Deep Neural Networks via Energy-Aware Pruning},
  year={2021},
  cdate={1609459200000},
  journal={CoRR},
  volume={abs/2103.10858},
  url={https://arxiv.org/abs/2103.10858},
}