Source code

001// Copyright (c) FIRST and other WPILib contributors.
002// Open Source Software; you can modify and/or share it under the terms of
003// the WPILib BSD license file in the root directory of this project.
004
005package edu.wpi.first.wpilibj;
006
007import edu.wpi.first.networktables.NetworkTableInstance;
008import edu.wpi.first.util.WPIUtilJNI;
009import edu.wpi.first.util.concurrent.Event;
010import edu.wpi.first.util.datalog.DataLog;
011import edu.wpi.first.util.datalog.IntegerLogEntry;
012import edu.wpi.first.util.datalog.StringLogEntry;
013import java.io.File;
014import java.io.IOException;
015import java.nio.file.Files;
016import java.nio.file.Path;
017import java.nio.file.Paths;
018import java.time.LocalDateTime;
019import java.time.ZoneId;
020import java.time.format.DateTimeFormatter;
021import java.util.Arrays;
022import java.util.Comparator;
023import java.util.Random;
024
025/**
026 * Centralized data log that provides automatic data log file management. It automatically cleans up
027 * old files when disk space is low and renames the file based either on current date/time or (if
028 * available) competition match number. The deta file will be saved to a USB flash drive if one is
029 * attached, or to /home/lvuser otherwise.
030 *
031 * <p>Log files are initially named "FRC_TBD_{random}.wpilog" until the DS connects. After the DS
032 * connects, the log file is renamed to "FRC_yyyyMMdd_HHmmss.wpilog" (where the date/time is UTC).
033 * If the FMS is connected and provides a match number, the log file is renamed to
034 * "FRC_yyyyMMdd_HHmmss_{event}_{match}.wpilog".
035 *
036 * <p>On startup, all existing FRC_TBD log files are deleted. If there is less than 50 MB of free
037 * space on the target storage, FRC_ log files are deleted (oldest to newest) until there is 50 MB
038 * free OR there are 10 files remaining.
039 *
040 * <p>By default, all NetworkTables value changes are stored to the data log.
041 */
042public final class DataLogManager {
043  private static DataLog m_log;
044  private static String m_logDir;
045  private static boolean m_filenameOverride;
046  private static final Thread m_thread;
047  private static final ZoneId m_utc = ZoneId.of("UTC");
048  private static final DateTimeFormatter m_timeFormatter =
049      DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss").withZone(m_utc);
050  private static boolean m_ntLoggerEnabled = true;
051  private static int m_ntEntryLogger;
052  private static int m_ntConnLogger;
053  private static StringLogEntry m_messageLog;
054
055  // if less than this much free space, delete log files until there is this much free space
056  // OR there are this many files remaining.
057  private static final long kFreeSpaceThreshold = 50000000L;
058  private static final int kFileCountThreshold = 10;
059
060  private DataLogManager() {}
061
062  static {
063    m_thread = new Thread(DataLogManager::logMain, "DataLogDS");
064    m_thread.setDaemon(true);
065  }
066
067  /** Start data log manager with default directory location. */
068  public static synchronized void start() {
069    start("", "", 0.25);
070  }
071
072  /**
073   * Start data log manager. The parameters have no effect if the data log manager was already
074   * started (e.g. by calling another static function).
075   *
076   * @param dir if not empty, directory to use for data log storage
077   */
078  public static synchronized void start(String dir) {
079    start(dir, "", 0.25);
080  }
081
082  /**
083   * Start data log manager. The parameters have no effect if the data log manager was already
084   * started (e.g. by calling another static function).
085   *
086   * @param dir if not empty, directory to use for data log storage
087   * @param filename filename to use; if none provided, the filename is automatically generated
088   */
089  public static synchronized void start(String dir, String filename) {
090    start(dir, filename, 0.25);
091  }
092
093  /**
094   * Start data log manager. The parameters have no effect if the data log manager was already
095   * started (e.g. by calling another static function).
096   *
097   * @param dir if not empty, directory to use for data log storage
098   * @param filename filename to use; if none provided, the filename is automatically generated
099   * @param period time between automatic flushes to disk, in seconds; this is a time/storage
100   *     tradeoff
101   */
102  public static synchronized void start(String dir, String filename, double period) {
103    if (m_log != null) {
104      return;
105    }
106    m_logDir = makeLogDir(dir);
107    m_filenameOverride = !filename.isEmpty();
108
109    // Delete all previously existing FRC_TBD_*.wpilog files. These only exist when the robot
110    // never connects to the DS, so they are very unlikely to have useful data and just clutter
111    // the filesystem.
112    File[] files =
113        new File(m_logDir)
114            .listFiles((d, name) -> name.startsWith("FRC_TBD_") && name.endsWith(".wpilog"));
115    if (files != null) {
116      for (File file : files) {
117        if (!file.delete()) {
118          System.err.println("DataLogManager: could not delete " + file);
119        }
120      }
121    }
122
123    m_log = new DataLog(m_logDir, makeLogFilename(filename), period);
124    m_messageLog = new StringLogEntry(m_log, "messages");
125    m_thread.start();
126
127    // Log all NT entries and connections
128    if (m_ntLoggerEnabled) {
129      startNtLog();
130    }
131  }
132
133  /**
134   * Log a message to the "messages" entry. The message is also printed to standard output (followed
135   * by a newline).
136   *
137   * @param message message
138   */
139  public static synchronized void log(String message) {
140    if (m_messageLog == null) {
141      start();
142    }
143    m_messageLog.append(message);
144    System.out.println(message);
145  }
146
147  /**
148   * Get the managed data log (for custom logging). Starts the data log manager if not already
149   * started.
150   *
151   * @return data log
152   */
153  public static synchronized DataLog getLog() {
154    if (m_log == null) {
155      start();
156    }
157    return m_log;
158  }
159
160  /**
161   * Get the log directory.
162   *
163   * @return log directory, or empty string if logging not yet started
164   */
165  public static synchronized String getLogDir() {
166    if (m_logDir == null) {
167      return "";
168    }
169    return m_logDir;
170  }
171
172  /**
173   * Enable or disable logging of NetworkTables data. Note that unlike the network interface for
174   * NetworkTables, this will capture every value change. Defaults to enabled.
175   *
176   * @param enabled true to enable, false to disable
177   */
178  public static synchronized void logNetworkTables(boolean enabled) {
179    boolean wasEnabled = m_ntLoggerEnabled;
180    m_ntLoggerEnabled = enabled;
181    if (m_log == null) {
182      start();
183      return;
184    }
185    if (enabled && !wasEnabled) {
186      startNtLog();
187    } else if (!enabled && wasEnabled) {
188      stopNtLog();
189    }
190  }
191
192  private static String makeLogDir(String dir) {
193    if (!dir.isEmpty()) {
194      return dir;
195    }
196
197    if (RobotBase.isReal()) {
198      try {
199        // prefer a mounted USB drive if one is accessible
200        Path usbDir = Paths.get("/u").toRealPath();
201        if (Files.isWritable(usbDir)) {
202          return usbDir.toString();
203        }
204      } catch (IOException ex) {
205        // ignored
206      }
207    }
208
209    return Filesystem.getOperatingDirectory().getAbsolutePath();
210  }
211
212  private static String makeLogFilename(String filenameOverride) {
213    if (!filenameOverride.isEmpty()) {
214      return filenameOverride;
215    }
216    Random rnd = new Random();
217    StringBuilder filename = new StringBuilder();
218    filename.append("FRC_TBD_");
219    for (int i = 0; i < 4; i++) {
220      filename.append(String.format("%04x", rnd.nextInt(0x10000)));
221    }
222    filename.append(".wpilog");
223    return filename.toString();
224  }
225
226  private static void startNtLog() {
227    NetworkTableInstance inst = NetworkTableInstance.getDefault();
228    m_ntEntryLogger = inst.startEntryDataLog(m_log, "", "NT:");
229    m_ntConnLogger = inst.startConnectionDataLog(m_log, "NTConnection");
230  }
231
232  private static void stopNtLog() {
233    NetworkTableInstance.stopEntryDataLog(m_ntEntryLogger);
234    NetworkTableInstance.stopConnectionDataLog(m_ntConnLogger);
235  }
236
237  private static void logMain() {
238    // based on free disk space, scan for "old" FRC_*.wpilog files and remove
239    {
240      File logDir = new File(m_logDir);
241      long freeSpace = logDir.getFreeSpace();
242      if (freeSpace < kFreeSpaceThreshold) {
243        // Delete oldest FRC_*.wpilog files (ignore FRC_TBD_*.wpilog as we just created one)
244        File[] files =
245            logDir.listFiles(
246                (dir, name) ->
247                    name.startsWith("FRC_")
248                        && name.endsWith(".wpilog")
249                        && !name.startsWith("FRC_TBD_"));
250        if (files != null) {
251          Arrays.sort(files, Comparator.comparingLong(File::lastModified));
252          int count = files.length;
253          for (File file : files) {
254            --count;
255            if (count < kFileCountThreshold) {
256              break;
257            }
258            long length = file.length();
259            if (file.delete()) {
260              freeSpace += length;
261              if (freeSpace >= kFreeSpaceThreshold) {
262                break;
263              }
264            } else {
265              System.err.println("DataLogManager: could not delete " + file);
266            }
267          }
268        }
269      }
270    }
271
272    int timeoutCount = 0;
273    boolean paused = false;
274    int dsAttachCount = 0;
275    int fmsAttachCount = 0;
276    boolean dsRenamed = m_filenameOverride;
277    boolean fmsRenamed = m_filenameOverride;
278    int sysTimeCount = 0;
279    IntegerLogEntry sysTimeEntry =
280        new IntegerLogEntry(
281            m_log, "systemTime", "{\"source\":\"DataLogManager\",\"format\":\"time_t_us\"}");
282
283    Event newDataEvent = new Event();
284    DriverStation.provideRefreshedDataEventHandle(newDataEvent.getHandle());
285    while (!Thread.interrupted()) {
286      boolean timedOut;
287      try {
288        timedOut = WPIUtilJNI.waitForObjectTimeout(newDataEvent.getHandle(), 0.25);
289      } catch (InterruptedException e) {
290        break;
291      }
292      if (Thread.interrupted()) {
293        break;
294      }
295      if (timedOut) {
296        timeoutCount++;
297        // pause logging after being disconnected for 10 seconds
298        if (timeoutCount > 40 && !paused) {
299          timeoutCount = 0;
300          paused = true;
301          m_log.pause();
302        }
303        continue;
304      }
305      // when we connect to the DS, resume logging
306      timeoutCount = 0;
307      if (paused) {
308        paused = false;
309        m_log.resume();
310      }
311
312      if (!dsRenamed) {
313        // track DS attach
314        if (DriverStation.isDSAttached()) {
315          dsAttachCount++;
316        } else {
317          dsAttachCount = 0;
318        }
319        if (dsAttachCount > 300) { // 6 seconds
320          LocalDateTime now = LocalDateTime.now(m_utc);
321          if (now.getYear() > 2000) {
322            // assume local clock is now synchronized to DS, so rename based on
323            // local time
324            m_log.setFilename("FRC_" + m_timeFormatter.format(now) + ".wpilog");
325            dsRenamed = true;
326          } else {
327            dsAttachCount = 0; // wait a bit and try again
328          }
329        }
330      }
331
332      if (!fmsRenamed) {
333        // track FMS attach
334        if (DriverStation.isFMSAttached()) {
335          fmsAttachCount++;
336        } else {
337          fmsAttachCount = 0;
338        }
339        if (fmsAttachCount > 250) { // 5 seconds
340          // match info comes through TCP, so we need to double-check we've
341          // actually received it
342          DriverStation.MatchType matchType = DriverStation.getMatchType();
343          if (matchType != DriverStation.MatchType.None) {
344            // rename per match info
345            char matchTypeChar;
346            switch (matchType) {
347              case Practice:
348                matchTypeChar = 'P';
349                break;
350              case Qualification:
351                matchTypeChar = 'Q';
352                break;
353              case Elimination:
354                matchTypeChar = 'E';
355                break;
356              default:
357                matchTypeChar = '_';
358                break;
359            }
360            m_log.setFilename(
361                "FRC_"
362                    + m_timeFormatter.format(LocalDateTime.now(m_utc))
363                    + "_"
364                    + DriverStation.getEventName()
365                    + "_"
366                    + matchTypeChar
367                    + DriverStation.getMatchNumber()
368                    + ".wpilog");
369            fmsRenamed = true;
370            dsRenamed = true; // don't override FMS rename
371          }
372        }
373      }
374
375      // Write system time every ~5 seconds
376      sysTimeCount++;
377      if (sysTimeCount >= 250) {
378        sysTimeCount = 0;
379        sysTimeEntry.append(WPIUtilJNI.getSystemTime(), WPIUtilJNI.now());
380      }
381    }
382    newDataEvent.close();
383  }
384}