Network.h

Go to the documentation of this file.

/*
 * This file belongs to the Galois project, a C++ library for exploiting
 * parallelism. The code is being released under the terms of the 3-Clause BSD
 * License (a copy is located in LICENSE.txt at the top-level directory).
 *
 * Copyright (C) 2018, The University of Texas at Austin. All rights reserved.
 * UNIVERSITY EXPRESSLY DISCLAIMS ANY AND ALL WARRANTIES CONCERNING THIS
 * SOFTWARE AND DOCUMENTATION, INCLUDING ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR ANY PARTICULAR PURPOSE, NON-INFRINGEMENT AND WARRANTIES OF
 * PERFORMANCE, AND ANY WARRANTY THAT MIGHT OTHERWISE ARISE FROM COURSE OF
 * DEALING OR USAGE OF TRADE.  NO WARRANTY IS EITHER EXPRESS OR IMPLIED WITH
 * RESPECT TO THE USE OF THE SOFTWARE OR DOCUMENTATION. Under no circumstances
 * shall University be liable for incidental, special, indirect, direct or
 * consequential damages or loss of profits, interruption of business, or
 * related expenses which may arise from use of Software or Documentation,
 * including but not limited to those resulting from defects in Software and/or
 * Documentation, or loss or inaccuracy of data of any kind.
 */

#ifndef GALOIS_RUNTIME_NETWORK_H
#define GALOIS_RUNTIME_NETWORK_H

#include "galois/runtime/Serialize.h"
#include "galois/runtime/MemUsage.h"
#include "galois/substrate/Barrier.h"

#include <mpi.h>

#include <cstdint>
#include <optional>
#include <tuple>

namespace galois::runtime {

using SendBuffer = SerializeBuffer;
using RecvBuffer = DeSerializeBuffer;

class NetworkInterface {
protected:
  void initializeMPI();

  void finalizeMPI();

  MemUsageTracker memUsageTracker;

  std::atomic<size_t> inflightSends;
  std::atomic<size_t> inflightRecvs;

#ifdef GALOIS_USE_BARE_MPI
public:
  inline void incrementMemUsage(uint64_t size) {
    memUsageTracker.incrementMemUsage(size);
  }
  inline void decrementMemUsage(uint64_t size) {
    memUsageTracker.decrementMemUsage(size);
  }
#endif

public:
  static uint32_t ID;
  static uint32_t Num;

  NetworkInterface();

  virtual ~NetworkInterface();

  void sendMsg(uint32_t dest, void (*recv)(uint32_t, RecvBuffer&),
               SendBuffer& buf);

  template <typename... Args>
  void sendSimple(uint32_t dest, void (*recv)(uint32_t, Args...),
                  Args... param);

  virtual void sendTagged(uint32_t dest, uint32_t tag, SendBuffer& buf,
                          int type = 0) = 0;

  void broadcast(void (*recv)(uint32_t, RecvBuffer&), SendBuffer& buf,
                 bool self = false);

  template <typename... Args>
  void broadcastSimple(void (*recv)(uint32_t, Args...), Args... param);

  void handleReceives();

  inline void resetMemUsage() { memUsageTracker.resetMemUsage(); }

  void reportMemUsage() const;

  virtual std::optional<std::pair<uint32_t, RecvBuffer>>
  recieveTagged(uint32_t tag, std::unique_lock<substrate::SimpleLock>* rlg,
                int type = 0) = 0;

  virtual void flush() = 0;

  virtual bool anyPendingSends() = 0;

  virtual bool anyPendingReceives() = 0;

  virtual unsigned long reportSendBytes() const = 0;
  virtual unsigned long reportSendMsgs() const = 0;
  virtual unsigned long reportRecvBytes() const = 0;
  virtual unsigned long reportRecvMsgs() const = 0;
  virtual std::vector<unsigned long> reportExtra() const = 0;
  virtual std::vector<std::pair<std::string, unsigned long>>
  reportExtraNamed() const = 0;
};

extern uint32_t evilPhase;

NetworkInterface& getSystemNetworkInterface();

namespace internal {
void destroySystemNetworkInterface();
} // namespace internal

uint32_t getHostID();

NetworkInterface& makeNetworkBuffered();

NetworkInterface& makeNetworkLCI();

substrate::Barrier& getHostBarrier();
substrate::Barrier& getHostFence();

// Implementations
namespace { // anon
template <typename... Args>
static void genericLandingPad(uint32_t src, RecvBuffer& buf) {
  void (*fp)(uint32_t, Args...);
  std::tuple<Args...> args;
  gDeserialize(buf, fp, args);
  std::apply([fp, src](Args... params) { fp(src, params...); }, args);
}

} // namespace

template <typename... Args>
void NetworkInterface::sendSimple(uint32_t dest,
                                  void (*recv)(uint32_t, Args...),
                                  Args... param) {
  SendBuffer buf;
  gSerialize(buf, (uintptr_t)recv, param...,
             (uintptr_t)genericLandingPad<Args...>);
  sendTagged(dest, 0, buf);
}

template <typename... Args>
void NetworkInterface::broadcastSimple(void (*recv)(uint32_t, Args...),
                                       Args... param) {
  SendBuffer buf;
  gSerialize(buf, (uintptr_t)recv, param...);
  broadcast(genericLandingPad<Args...>, buf, false);
}

} // namespace galois::runtime
#endif