Filter tensorflow event files

filter_events.py

# -*- coding: utf-8 -*-
from __future__ import print_function

import os
import sys
import argparse
import tqdm

import tensorflow as tf

def parse_arguments():
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('--event', help='event file', required=True)

    return parser.parse_args()

def main(args):
    out_path = os.path.join(os.path.dirname(args.event), 'filtered_events')
    if not os.path.exists(out_path):
       os.makedirs(out_path)

    writer = tf.summary.FileWriter(out_path)

    total = None
    ## Pre-compute total number of events (takes lot of time)
    # total = 0
    # for event in tf.train.summary_iterator(args.event):
    #     total += 1

    # Working with Summaries and Events: https://stackoverflow.com/a/45023082/1705970
    # https://github.com/tensorflow/tensorflow/blob/r1.10/tensorflow/core/util/event.proto
    # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto
    for event in tqdm.tqdm(tf.train.summary_iterator(args.event), total=total):
        event_type = event.WhichOneof('what')
        if event_type != 'summary':
            writer.add_event(event)
        else:
            wall_time = event.wall_time
            step = event.step

            # possible types: simple_value, image, histo, audio
            filtered_values = [value for value in event.summary.value if value.HasField('simple_value')]
            summary = tf.Summary(value=filtered_values)

            filtered_event = tf.summary.Event(summary=summary,
                                              wall_time=wall_time,
                                              step=step)
            writer.add_event(filtered_event)
    writer.close()
    return 0

if __name__ == '__main__':
    args = parse_arguments()
    sys.exit(main(args))

"updated" to work with tensorflow 2.11.0 by using the compat layer:

# -*- coding: utf-8 -*-
from __future__ import print_function      
# iterate through a tensorboard events file and save only the scalars but not the image summaries
# into a new such file. That should reduce the file size.
                                                                                                     
# Original file by srycjon, CC-BY-SA                                                                 # https://gist.githubusercontent.com/serycjon/c9ad58ecc3176d87c49b69b598f4d6c6/raw/37f62889c6315ae830
354902ae01d7069251c661/filter_events.py                                                              
# https://stackoverflow.com/questions/42774317/remove-data-from-tensorboard-event-files-to-make-them-
smaller                                                                                              
                                                                                                     
import os                                                                                            
import sys                                        
import argparse       
import tqdm 

import tensorflow as tf                           
                                                  
def parse_arguments():  
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('--event', help='event file', required=True)
    parser.add_argument('--out-event', help='output event file ', required=False, default=None)

    return parser.parse_args()

def main(args):
    if args.out_event is None:
        out_path = os.path.join(os.path.dirname(args.event), 'filtered_events')
    else:
        out_path = args.out_event
    if not os.path.exists(out_path):
       os.makedirs(out_path)

    with tf.compat.v1.Graph().as_default():
        writer = tf.compat.v1.summary.FileWriter(out_path)

        total = None
        ## Pre-compute total number of events (takes lot of time)
        # total = 0
        # for event in tf.train.summary_iterator(args.event):
        #     total += 1

        # Working with Summaries and Events: https://stackoverflow.com/a/45023082/1705970
        # https://github.com/tensorflow/tensorflow/blob/r1.10/tensorflow/core/util/event.proto
        # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto
        for event in tqdm.tqdm(tf.compat.v1.train.summary_iterator(args.event), total=total):
            event_type = event.WhichOneof('what')
            if event_type != 'summary':
                writer.add_event(event)
            else:
                wall_time = event.wall_time
                step = event.step

                # possible types: simple_value, image, histo, audio
                filtered_values = [value for value in event.summary.value if value.HasField('simple_value')]
                summary = tf.compat.v1.Summary(value=filtered_values)

                filtered_event = tf.compat.v1.summary.Event(summary=summary,
                                                  wall_time=wall_time,
                                                  step=step)
                writer.add_event(filtered_event)
        writer.close()
    return 0

if __name__ == '__main__':
    args = parse_arguments()
    sys.exit(main(args))

No idea how much there might be in terms of new features that this does not support because it is based on the v1 API though. But the scalars match up between the old and the new file, so thanks a lot for providing this gist!

luckily I don't have to use tensorflow any more :D so I have no idea about new features.