• Main Page
  • Related Pages
  • Modules
  • Data Structures
  • Files
  • File List
  • Globals

sst/elements/SS_router/SS_router/SS_router.h

00001 // Copyright 2009-2010 Sandia Corporation. Under the terms
00002 // of Contract DE-AC04-94AL85000 with Sandia Corporation, the U.S.
00003 // Government retains certain rights in this software.
00004 // 
00005 // Copyright (c) 2009-2010, Sandia Corporation
00006 // All rights reserved.
00007 // 
00008 // This file is part of the SST software package. For license
00009 // information, see the LICENSE file in the top level directory of the
00010 // distribution.
00011 
00012 
00013 /* Copyright 2007 Sandia Corporation. Under the terms
00014  of Contract DE-AC04-94AL85000 with Sandia Corporation, the U.S.
00015  Government retains certain rights in this software.
00016 
00017  Copyright (c) 2007, Sandia Corporation
00018  All rights reserved.
00019 
00020  This file is part of the SST software package. For license
00021  information, see the LICENSE file in the top level directory of the
00022  distribution. */
00023 
00024 
00025 #ifndef ED_ROUTER_H
00026 #define ED_ROUTER_H
00027 
00028 #include <map>
00029 #include <list>
00030 #include <deque>
00031 #include <string>
00032 #include <vector>
00033 
00034 #include <queue>
00035 #include <sst/core/component.h>
00036 #include <sst/core/log.h>
00037 #include <sst/core/link.h>
00038 
00039 #include "SS_network.h"
00040 
00041 using namespace SST;
00042 
00043 #define SS_ROUTER_DBG 1 
00044 #ifndef SS_ROUTER_DBG
00045 #define SS_ROUTER_DBG 0
00046 #endif
00047 
00048 #include "SS_network.h"
00049 
00050 using namespace std;
00051 
00052 #define DBprintf(fmt,args...) \
00053     m_dbg.write( "%s():%d: "fmt, __FUNCTION__, __LINE__, ##args)
00054 
00055 #define cycle() m_cycle
00056 
00057 enum rtrEventType { iLCB_internalXferDone, InQ_tailXferDone, InQ_headXferDone,
00058                     OutQ_tailXferDone, OutQ_headXferDone, oLCB_internalXferDone,
00059                     oLCB_externalXferDone, Debug
00060                   };
00061 
00062 typedef struct rtrEvent_s rtrEvent_t;
00063 
00064 //: router parcel
00065 // wrapper for a parcel while it lives in the router
00066 //!SEC:EDSim
00067 class rtrP
00068 {
00069 public:
00070     RtrEvent *event;
00071     int ilink;
00072     int olink;
00073     int ivc, ovc;
00074     int flits;
00075 };
00076 
00077 //: an internal router event
00078 // wrapper for a parcel while it waits in a priority queue
00079 //!SEC:EDSim
00080 struct rtrEvent_s {
00081     long long unsigned cycle;
00082     rtrEventType type;
00083     rtrP *rp;
00084 };
00085 
00086 //: priority function to use for router Events
00087 //!SEC:EDSim
00088 inline bool rtrEvent_gt (rtrEvent_t* a, rtrEvent_t *b) {
00089     return (a->cycle > b->cycle);
00090 }
00091 
00092 //: Pool of router Events
00093 // used to avoid many memory allocates
00094 //!SEC:EDSim
00095 class rtrEventPool {
00096 private:
00097     vector<rtrEvent_t*> thePool;
00098     int count;
00099 public:
00100     rtrEventPool() {
00101         rtrEvent_t *event;
00102         for (int i = 0; i < 10; i++) {
00103             event = new rtrEvent_t;
00104             thePool.push_back(event);
00105         }
00106         count = 10;
00107     }
00108 
00109     rtrEvent_t *getEvent() {
00110         rtrEvent_t *event;
00111         if (count > 0) {
00112             event = thePool.back();
00113             thePool.pop_back();
00114             count--;
00115         } else {
00116             event = new rtrEvent_t;
00117             thePool.push_back(event);
00118             event = new rtrEvent_t;
00119             count++;
00120         }
00121         return event;
00122     }
00123 
00124     void returnEvent(rtrEvent_t* e) {
00125         thePool.push_back(e);
00126         count++;
00127     }
00128 };
00129 
00130 
00131 //: Pool of router parcel wrappers
00132 // used to avoid many memory allocates in the long run
00133 //!SEC:EDSim
00134 class rtrPPool {
00135 private:
00136     vector<rtrP*> thePool;
00137     int count;
00138 public:
00139     rtrPPool() {
00140         rtrP *rp;
00141         for (int i = 0; i < 10; i++) {
00142             rp = new rtrP;
00143             thePool.push_back(rp);
00144         }
00145         count = 10;
00146     }
00147 
00148     rtrP *getRp() {
00149         rtrP *rp;
00150         if (count > 0) {
00151             rp = thePool.back();
00152             thePool.pop_back();
00153             count--;
00154         } else {
00155             rp = new rtrP;
00156             thePool.push_back(rp);
00157             rp = new rtrP;
00158             count++;
00159         }
00160         return rp;
00161     }
00162 
00163     void returnRp(rtrP* rp) {
00164         thePool.push_back(rp);
00165         count++;
00166     }
00167 };
00168 
00169 //: 3D router
00170 // models a 3D router in a toroid configuration with 4 virtual channels
00171 //!SEC:EDSim
00172 class SS_router : public Component
00173 {
00174     bool findRoute( int destNid, int inputVC, int inputDir,
00175                     int& outVC, int& outDir );
00176     int findOutputDir( int destNid );
00177     int findOutputVC( int inVC, int inDir, int outDir );
00178     int dimension( int dir );
00179     bool iAmDateline( int dimension );
00180     int changeVC( int vc );
00181     int calcXPosition( int nodeNumber, int xSize, int ySize, int zSize );
00182     int calcYPosition( int nodeNumber, int xSize, int ySize, int zSize );
00183     int calcZPosition( int nodeNumber, int xSize, int ySize, int zSize );
00184     int calcDirection( int src, int dst, int size );
00185 
00186     vector<char> m_routingTableV; 
00187     vector<bool> m_datelineV;
00188 
00189 
00190     int neighbor[ROUTER_NUM_LINKS];
00191     int neighborID (int dir) {
00192         return neighbor[dir];
00193     }
00194 
00195     inline int NODE_ID(int x, int y, int z,Network *net) {
00196         return z * net->xDimSize() * net->yDimSize() + y * net->xDimSize() + x;
00197     }
00198 
00199     void find_neighbors(Network *net, int x, int y, int z) {
00200 
00201         DBprintf("located at (%d, %d, %d)\n", x, y, z);
00202 
00203         int posx,posy,posz,negx,negy,negz;
00204         posx = (x + 1) % net->xDimSize();
00205         posy = (y + 1) % net->yDimSize();
00206         posz = (z + 1) % net->zDimSize();
00207         negx = (x == 0) ? net->xDimSize() - 1 : (x - 1);
00208         negy = (y == 0) ? net->yDimSize() - 1 : (y - 1);
00209         negz = (z == 0) ? net->zDimSize() - 1 : (z - 1);
00210 
00211         neighbor[LINK_NEG_X] = NODE_ID(negx,y,z, net);
00212         neighbor[LINK_POS_X] = NODE_ID(posx,y,z, net);
00213 
00214         neighbor[LINK_NEG_Y] = NODE_ID(x,negy,z, net);
00215         neighbor[LINK_POS_Y] = NODE_ID(x,posy,z, net);
00216 
00217         neighbor[LINK_NEG_Z] = NODE_ID(x,y,negz, net);
00218         neighbor[LINK_POS_Z] = NODE_ID(x,y,posz, net);
00219         DBprintf("X %d %d\n", neighbor[LINK_NEG_X], neighbor[LINK_POS_X]);
00220         DBprintf("Y %d %d\n", neighbor[LINK_NEG_Y], neighbor[LINK_POS_Y]);
00221         DBprintf("Z %d %d\n", neighbor[LINK_NEG_Z], neighbor[LINK_POS_Z]);
00222     }
00223 
00224     friend class oLCB_t;
00225 
00226 private:
00227     double overheadMultP;
00228 protected:
00229     //: queue of internal router events
00230     vector<rtrEvent_t*> rtrEventQ;
00231 
00232     //: pool of unused allocated events to draw from
00233     rtrEventPool eventPool;
00234     //: pool of unused allocated router parcel wrappers
00235     rtrPPool rpPool;
00236     //: router ID, same as node ID
00237     int routerID;
00238     //: latency to get a packet
00239     int iLCBLat;
00240     //: latency to send a packet
00241     int oLCBLat;
00242     //: latency to route a packet
00243     int routingLat;
00244     //: queue access latency
00245     int iQLat;
00246     //: max output queue size in flits
00247     int rtrOutput_maxQSize_flits[ROUTER_NUM_LINKS + 1];
00248     //: max input queue size in flits
00249     int rtrInput_maxQSize_flits[ROUTER_NUM_LINKS + 1];
00250     //: number of flits which can fit into an LCB
00251     int oLCB_maxSize_flits;
00252 
00253 
00254     //: Routing table
00255     // It maps a destination node ID and incoming virtual channel
00256     // To an outgoing link and virtual channel
00257     // The virtual channel will always be the same unless
00258     // crossing a dateline
00259     typedef map< pair<int, int>, pair<int, int> > routingTable_t;
00260     //: routing table structure
00261     routingTable_t routingTable;
00262 
00263 public:
00264     void updateToken_flits (int dir, int vc, int flits );
00265     void returnToken_flits (int dir, int vc, int flits );
00266 
00267 protected:
00268     //: a link between routers, used to hold ID neighbor info and pass tokens
00269     struct netlink {
00270         Link *link;
00271         int   dir;
00272     };
00273     //: netlink pointer
00274     typedef netlink* link_t;
00275 
00276     //: RX Network links each of the six directions
00277     link_t rx_netlinks[ROUTER_NUM_LINKS];
00278     //: TX Network links each of the six directions
00279     link_t tx_netlinks[ROUTER_NUM_LINKS];
00280     //: pointers to neighbors who may send us packets
00281     // which is used when packets arrive from external sources, to check for a valid sender
00282     map <Link*, int> rxNeighbor;
00283 
00284     //: count packets sent from this node on each link
00285     int txCount[ROUTER_NUM_OUTQS];
00286     //: count packets received at this node on each link
00287     int rxCount[ROUTER_NUM_INQS];
00288     //: queue structure used for all input and output queues
00289     typedef deque<rtrP*> packetQ;
00290 
00291     //: input queue structure
00292     // contains a busy flag, 4 sub-queus for the virtual channels, a buffer for keeping
00293     // track of the queues that were skipped in a RR, and size of each queue in flits
00294     // which is used to reserve size to flow control between LCB and input queues
00295     struct inQ_t
00296     {
00297         bool head_busy;
00298         int link, vc_rr, ready_vcQs;
00299         queue<int> skipQs;
00300         int size_flits[ROUTER_NUM_VCS];
00301         packetQ vcQ[ROUTER_NUM_VCS];
00302         bool ready () {
00303             return ((ready_vcQs > 0) && (!head_busy));
00304         }
00305     };
00306     //: One input queue set per rx-link
00307     inQ_t inputQ[ROUTER_NUM_INQS];
00308 
00309     //: number of input queues which are ready to try and dequeue something
00310     //int ready_inQs_count;
00311     bool ready_inQ;
00312 
00313     //: ouput queue structure
00314     // contains 4 sub-queues for virtual channels, and size of each queue in flits
00315     // which is used to reserve size to flow control between input and output queues
00316     struct outQ_t
00317     {
00318         packetQ vcQ[ROUTER_NUM_VCS];
00319         int size_flits[ROUTER_NUM_VCS];
00320     };
00321 
00322     //: One output queue per tx-link per rx-link
00323     outQ_t outputQ[ROUTER_NUM_OUTQS][ROUTER_NUM_INQS];
00324 
00325     //: out LCB structure
00326     // contains round robin information as well as packet data
00327     struct oLCB_t
00328     {
00329         int size_flits;
00330         bool external_busy;
00331         bool internal_busy;
00332         int link;
00333         int vc_rr, ready_vc_count;
00334         queue<int> skipped_vcs;
00335         int ready_outQ_count[ROUTER_NUM_VCS];
00336         int ilink_rr[ROUTER_NUM_VCS];
00337         int vcTokens[ROUTER_NUM_VCS];
00338         int stall4tokens[ROUTER_NUM_VCS];
00339         packetQ dataQ;
00340 
00341         bool readyXfer() {
00342             return (!dataQ.empty() && !external_busy);
00343         }
00344         bool readyInternal() {
00345             return ((ready_vc_count > 0) && !internal_busy);
00346         }
00347 
00348     };
00349 
00350     //: in LCB structure
00351     // contains round robin information as well as packet data
00352     struct iLCB_t
00353     {
00354         int size_flits;
00355         bool internal_busy;
00356         int link;
00357         packetQ dataQ;
00358         bool readyInternal() {
00359             return (!dataQ.empty() && !internal_busy);
00360         }
00361     };
00362 
00363     //: output LCBs for each link plus one for the router port
00364     oLCB_t outLCB[ROUTER_NUM_OUTQS];
00365     //: input LCBs for each link plus one for the router port
00366     iLCB_t inLCB[ROUTER_NUM_INQS];
00367 
00368     //: number of out LCBs ready to move data
00369     //int ready_oLCB_count;
00370     bool ready_oLCB;
00371     //: number of in LCBs ready to move data
00372     //int ready_iLCB_count;
00373     bool ready_iLCB;
00374 
00375     Network *network;
00376 
00377     vector<Link*> linkV;
00378 
00379 /*     void handleNicParcel(Event*); */
00380     void handleParcel(Event*,int dir);
00381     bool route(rtrP*);
00382 
00383     void InLCB ( RtrEvent *e, int ilink, int ivc, int flits);
00384 
00385     bool LCBtoInQ_start(rtrP *rp);
00386     void LCBtoInQ_readyNext(rtrP *rp);
00387     void LCBtoInQ_done (rtrP *rp, int ivc, int ilink);
00388     void InQtoOutQ_start (rtrP *rp);
00389     void InQtoOutQ_readyNext (rtrP *rp);
00390     void InQtoOutQ_done (rtrP *rp, int ovc, int ilink, int olink);
00391     void OutQtoLCB_start (rtrP *rp);
00392     void OutQtoLCB_readyNext (rtrP *rp, int olink, int ilink, int ovc, int flits);
00393     void OutQtoLCB_done (rtrP *rp);
00394     void LCBxfer_start (int dir);
00395     void LCBxfer_readyNext (int dir);
00396     void LCBxfer_done (rtrP *rp, int olink, int flits);
00397 
00398     void arbitrateOutToLCB ();
00399     void arbitrateInToOut ();
00400     void iLCBtoIn ();
00401 
00402     void setup();
00403     void dumpStats(FILE *fp);
00404     int Finish();
00405 
00406     void setupRoutingTable ( Params_t, int nodes, int xDim, int yDim, int zDim);
00407     void setVCRoutes ( int node, int dir, bool *crossDateline );
00408 
00409     void advanceEventQ();
00410     void DebugEvent ();
00411     //: if using the debug event, this is the number of cycles between successive events
00412     int debug_interval;
00413     bool dumpTables;
00414 
00415 public:
00416 
00417     SS_router ( ComponentId_t id, Params_t& params );
00418 
00419 private:
00420     SS_router();
00421     SS_router( const SS_router& );
00422     ~SS_router() {
00423         ;
00424     }
00425 
00426     // Keeps track of when router is on the clock list
00427     bool currently_clocking;
00428     std::string frequency;
00429     Clock::Handler<SS_router>* clock_handler;
00430 
00431     //: simulate one pretic
00432     // advance the event queue, try to move data from in LCBs to input queues,
00433     // try to move data from input queues to output queues, and
00434     // try to move data from output queues to out LCBs
00435     bool clock( Cycle_t cycle )
00436     {
00437 //      clock_count++;
00438         bool work_done = false;
00439         m_cycle = cycle;
00440         if (!rtrEventQ.empty()) {
00441             advanceEventQ();
00442             work_done = true;
00443         }
00444         //if (ready_oLCB_count > 0)
00445         //if (ready_oLCBsleepchk())
00446         if (ready_oLCB) {
00447             arbitrateOutToLCB ();
00448             work_done = true;
00449         }
00450         //if (ready_inQs_count > 0)
00451         if (ready_inQ) {
00452             arbitrateInToOut ();
00453             work_done = true;
00454         }
00455         //if (ready_iLCB_count > 0)
00456         if (ready_iLCB) {
00457             iLCBtoIn();
00458             work_done = true;
00459         }
00460         if ( !work_done ) {
00461             unregisterClock(defaultTimeBase,clock_handler);
00462             currently_clocking = false;
00463         }
00464         return true; // KBW: hopefully, this is essentially meaningless
00465     }
00466 
00467     //: string values for link direction, only used for debugging and outputting link traffic data
00468     char LINK_DIR_STR[6][8];
00469     //: return router ID
00470     int getID() {
00471         return routerID;
00472     }
00473 
00474     int clock_count;
00475     
00476     void dumpTable( FILE* fp);
00477     void dumpState();
00478 
00479     void txlinkTo ( Link* neighbor, int dir);
00480     bool checkLinks ();
00481 
00482     Cycle_t                 m_cycle;
00483     bool                    m_print_info;
00484     Log< SS_ROUTER_DBG >&   m_dbg;
00485     Log<>&                  m_log;
00486 };
00487 
00488 #include "SS_router-inline.h"
00489 
00490 #endif

Generated on Fri Oct 22 2010 11:02:28 for SST by  doxygen 1.7.1